Skip to content
Commits on Source (7)
# NCBI External Developer Release:
## NCBI VDB 2.9.3
**October 17, 2018**
**kns**: added possibility to skip server's certificate validation
**kns**: expect to receive HTTP status 200 when sending range-request that includes the whole file
**vdb**: fixed a bug in accessing pagemap process request for cursors which do not have pagemap thread running
## NCBI VDB 2.9.2-1
**July 31, 2018**
**vdb**: restored lost exported attribute on several entries in the private API
......
......@@ -23,4 +23,4 @@
# ===========================================================================
# NCBI-VDB and library version
VERSION = 2.9.2
VERSION = 2.9.3
ncbi-vdb (2.9.3+dfsg-1) unstable; urgency=medium
* New upstream version
* debhelper 12
* Standards-Version: 4.3.0
-- Andreas Tille <tille@debian.org> Fri, 11 Jan 2019 23:15:24 +0100
ncbi-vdb (2.9.2-1+dfsg-1) unstable; urgency=medium
* New upstream version
......
......@@ -3,7 +3,7 @@ Maintainer: Debian Med Packaging Team <debian-med-packaging@lists.alioth.debian.
Uploaders: Andreas Tille <tille@debian.org>
Section: science
Priority: optional
Build-Depends: debhelper (>= 11~),
Build-Depends: debhelper (>= 12~),
d-shlibs (>= 0.82),
libbz2-dev,
libxml2-dev,
......@@ -14,7 +14,7 @@ Build-Depends: debhelper (>= 11~),
libmbedtls-dev,
zlib1g-dev,
default-jdk-headless
Standards-Version: 4.2.1
Standards-Version: 4.3.0
Vcs-Browser: https://salsa.debian.org/med-team/ncbi-vdb
Vcs-Git: https://salsa.debian.org/med-team/ncbi-vdb.git
Homepage: https://github.com/ncbi/ncbi-vdb
......
......@@ -68,20 +68,23 @@ Description: Name change of function probably a consequence of using Debian
MBEDTLS_SSL_IS_CLIENT,
MBEDTLS_SSL_TRANSPORT_STREAM,
MBEDTLS_SSL_PRESET_DEFAULT );
@@ -457,9 +457,9 @@ rc_t tlsg_setup ( KTLSGlobals * self )
return rc;
@@ -458,12 +458,12 @@ rc_t tlsg_setup ( KTLSGlobals * self )
}
- vdb_mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
/* turn off certificate validation when self -> allow_all_certs == true */
- vdb_mbedtls_ssl_conf_authmode( &self -> config,
+ mbedtls_ssl_conf_authmode( &self -> config,
self -> allow_all_certs ? MBEDTLS_SSL_VERIFY_OPTIONAL
: MBEDTLS_SSL_VERIFY_REQUIRED );
- vdb_mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
- vdb_mbedtls_ssl_conf_rng( &self -> config, vdb_mbedtls_ctr_drbg_random, &self -> ctr_drbg );
+ mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
+ mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
+ mbedtls_ssl_conf_rng( &self -> config, mbedtls_ctr_drbg_random, &self -> ctr_drbg );
return 0;
}
@@ -494,7 +494,7 @@ static int set_threshold ( const KConfig
/* We need that to be sure that we are free to call
* vdb_mbedtls_ssl_conf_authmode () next time when
@@ -507,7 +507,7 @@ static int set_threshold ( const KConfig
}
if ( set )
......@@ -90,7 +93,7 @@ Description: Name change of function probably a consequence of using Debian
return ( int ) threshold;
}
@@ -508,13 +508,13 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tls
@@ -521,13 +521,13 @@ rc_t KTLSGlobalsInit ( KTLSGlobals * tls
assert ( tlsg != NULL );
assert ( kfg != NULL );
......@@ -109,7 +112,7 @@ Description: Name change of function probably a consequence of using Debian
rc = tlsg_seed_rng ( tlsg );
if ( rc == 0 )
@@ -533,10 +533,10 @@ void KTLSGlobalsWhack ( KTLSGlobals * se
@@ -546,10 +546,10 @@ void KTLSGlobalsWhack ( KTLSGlobals * se
{
assert ( self != NULL );
......@@ -124,7 +127,7 @@ Description: Name change of function probably a consequence of using Debian
memset ( self, 0, sizeof * self );
}
@@ -609,8 +609,8 @@ static
@@ -637,8 +637,8 @@ static
void KTLSStreamDestroy ( KTLSStream *self )
{
/* tear down all of the stuff created during Make */
......@@ -135,7 +138,7 @@ Description: Name change of function probably a consequence of using Debian
/* release the ciphertext object */
KStreamRelease ( self -> ciphertext );
@@ -652,7 +652,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
@@ -680,7 +680,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
while ( 1 )
{
/* read through TLS library */
......@@ -144,7 +147,7 @@ Description: Name change of function probably a consequence of using Debian
/* no error */
if ( ret >= 0 )
@@ -682,7 +682,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
@@ -710,7 +710,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
{
/* The ret is anything other than the following 3, then the ssl context becomes
* becomes unusable and should either be freed or call
......@@ -153,7 +156,7 @@ Description: Name change of function probably a consequence of using Debian
* must be closed
*/
case MBEDTLS_ERR_SSL_WANT_READ:
@@ -694,7 +694,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
@@ -722,7 +722,7 @@ rc_t CC KTLSStreamRead ( const KTLSStrea
* is initiating a new connection using the same source port.
* You can either treat that as a connection close and wait
* for the client to resend a ClientHello, or directly
......@@ -162,7 +165,7 @@ Description: Name change of function probably a consequence of using Debian
* context (as it has beeen reset internally). Either way, you
* should make sure this is seen by the application as a new
* connection: application state, if any, should be reset, and
@@ -758,7 +758,7 @@ rc_t CC KTLSStreamWrite ( KTLSStream * s
@@ -786,7 +786,7 @@ rc_t CC KTLSStreamWrite ( KTLSStream * s
* We expect to be called through KStreamWriteAll that will
* avoid the issue above.
*/
......@@ -171,7 +174,7 @@ Description: Name change of function probably a consequence of using Debian
/* no error */
if ( ret >= 0 )
@@ -873,7 +873,7 @@ int CC ktls_net_send ( void *ctx, const
@@ -901,7 +901,7 @@ int CC ktls_net_send ( void *ctx, const
return ( int ) num_writ;
}
......@@ -180,7 +183,7 @@ Description: Name change of function probably a consequence of using Debian
static
int CC ktls_net_recv ( void *ctx, unsigned char *buf, size_t len )
{
@@ -918,7 +918,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
@@ -946,7 +946,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
assert ( self -> mgr != NULL );
tlsg = & self -> mgr -> tlsg;
......@@ -189,7 +192,7 @@ Description: Name change of function probably a consequence of using Debian
if ( ret != 0 )
{
rc_t rc = RC ( rcKrypto, rcSocket, rcFormatting, rcEncryption, rcFailed );
@@ -959,7 +959,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
@@ -987,7 +987,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
return rc;
}
......@@ -198,7 +201,7 @@ Description: Name change of function probably a consequence of using Debian
if ( hostz != host )
StringWhack ( hostz );
@@ -977,7 +977,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
@@ -1005,7 +1005,7 @@ rc_t ktls_ssl_setup ( KTLSStream *self,
}
......@@ -207,7 +210,7 @@ Description: Name change of function probably a consequence of using Debian
return 0;
}
@@ -989,7 +989,7 @@ rc_t ktls_handshake ( KTLSStream *self )
@@ -1017,7 +1017,7 @@ rc_t ktls_handshake ( KTLSStream *self )
STATUS ( STAT_QA, "Performing SSL/TLS handshake...\n" );
......@@ -216,7 +219,7 @@ Description: Name change of function probably a consequence of using Debian
while ( ret != 0 )
{
if ( ret != MBEDTLS_ERR_SSL_WANT_READ &&
@@ -1021,11 +1021,11 @@ rc_t ktls_handshake ( KTLSStream *self )
@@ -1049,11 +1049,11 @@ rc_t ktls_handshake ( KTLSStream *self )
if ( ret == MBEDTLS_ERR_X509_CERT_VERIFY_FAILED )
{
......@@ -230,7 +233,7 @@ Description: Name change of function probably a consequence of using Debian
PLOGMSG ( klogSys, ( klogSys
, "mbedtls_ssl_get_verify_result returned $(flags) ( $(info) )"
@@ -1039,7 +1039,7 @@ rc_t ktls_handshake ( KTLSStream *self )
@@ -1067,7 +1067,7 @@ rc_t ktls_handshake ( KTLSStream *self )
return rc;
}
......@@ -239,7 +242,7 @@ Description: Name change of function probably a consequence of using Debian
}
return 0;
@@ -1075,7 +1075,7 @@ rc_t KTLSStreamMake ( KTLSStream ** objp
@@ -1103,7 +1103,7 @@ rc_t KTLSStreamMake ( KTLSStream ** objp
obj -> mgr = mgr;
STATUS ( STAT_PRG, "%s - initializing tls wrapper\n", __func__ );
......@@ -248,7 +251,7 @@ Description: Name change of function probably a consequence of using Debian
* objp = obj;
return 0;
@@ -1217,13 +1217,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACer
@@ -1245,13 +1245,13 @@ LIB_EXPORT rc_t CC KTLSStreamVerifyCACer
rc = RC ( rcKrypto, rcToken, rcValidating, rcSelf, rcNull );
else if ( ! self -> mgr -> tlsg . allow_all_certs )
{
......
......@@ -50,7 +50,7 @@
#include <stdio.h> /* fprintf */
#include <string.h> /* memset */
#define TOOLKIT "sratoolkit2_9_2"
#define TOOLKIT "sratoolkit2_9_3"
/******************************************************************************/
......
......@@ -24,4 +24,4 @@
*
*/
#define LIBKDB_VERS 0x0207001B
#define LIBKDB_VERS 0x0207001C
......@@ -28,7 +28,7 @@
/* Version of current SRA Toolkit Release */
#define RELEASE_VERS 0x02090002
#define RELEASE_VERS 0x02090003
/* Type of Version of current SRA Toolkit Release is one of:
......
......@@ -250,19 +250,38 @@ otherwise we are going to hit "Apache return HTTP headers twice" bug */
TRACE ( "KClientHttpResultStatus ( rslt, & http_status, NULL, 0, NULL ); failed: rc=%u\n", rc );
else
{
uint64_t start_pos = 0;
size_t result_size = 0;
bool have_size = false;
switch ( * http_status )
{
case 200:
/* We requested Bytes Range but got 200:
accept it what the whole file was requested */
have_size = KClientHttpResultSize ( rslt,
&result_size );
if ( pos != 0 || ! have_size
|| result_size > bsize )
{
rc = RC ( rcNS, rcFile, rcReading,
rcData, rcUnexpected );
TRACE ( "KClientHttpResultStatus ( rslt, "
"& http_status, NULL, 0, NULL ); unexpected status=%d\n", * http_status );
break;
}
/* no break here, now read the file */
case 206:
{
uint64_t start_pos;
size_t result_size;
/* don't need retries now */
proxy_retries = 0;
/* extract actual amount being returned by server */
rc = KClientHttpResultRange ( rslt, &start_pos, &result_size );
if ( rc != 0 || start_pos != pos || result_size != bsize )
if ( * http_status == 206 ) {
/* get result range when 206 was returned,
we got it already when status == 200 */
rc = KClientHttpResultRange ( rslt,
&start_pos, &result_size );
if ( rc != 0 || start_pos != pos
|| result_size != bsize )
{
if ( rc != 0 )
TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: rc=%u\n", rc );
......@@ -273,9 +292,13 @@ otherwise we are going to hit "Apache return HTTP headers twice" bug */
if ( result_size != bsize )
TRACE ( "KClientHttpResultRange ( rslt, & start_pos, & result_size ); failed: result_size=%lu != bsize=%lu\n", result_size, bsize );
}
break;
}
}
else
{
/* read the response for partial file requests
or when the whole file was returned */
KStream *response;
rc = KClientHttpResultGetInputStream ( rslt, &response );
......@@ -318,7 +341,6 @@ otherwise we are going to hit "Apache return HTTP headers twice" bug */
}
}
break;
}
case 403:
case 404:
......
......@@ -64,6 +64,8 @@ struct KTLSGlobals
mbedtls_ssl_config config;
bool allow_all_certs;
bool safe_to_modify_ssl_config; /* needed for SetAllowAllCerts () */
};
/* Init
......
......@@ -457,10 +457,23 @@ rc_t tlsg_setup ( KTLSGlobals * self )
return rc;
}
vdb_mbedtls_ssl_conf_authmode( &self -> config, MBEDTLS_SSL_VERIFY_REQUIRED );
/* turn off certificate validation when self -> allow_all_certs == true */
vdb_mbedtls_ssl_conf_authmode( &self -> config,
self -> allow_all_certs ? MBEDTLS_SSL_VERIFY_OPTIONAL
: MBEDTLS_SSL_VERIFY_REQUIRED );
vdb_mbedtls_ssl_conf_ca_chain( &self -> config, &self -> cacert, NULL );
vdb_mbedtls_ssl_conf_rng( &self -> config, vdb_mbedtls_ctr_drbg_random, &self -> ctr_drbg );
/* We need that to be sure that we are free to call
* vdb_mbedtls_ssl_conf_authmode () next time when
* KNSManagerSetAllowAllCerts () will be called
*
* Because smart special design we do not need to add
* special code to deinitialize that variable.
*/
self -> safe_to_modify_ssl_config = true;
return 0;
}
......@@ -553,6 +566,21 @@ LIB_EXPORT rc_t CC KNSManagerSetAllowAllCerts ( KNSManager *self, bool allow_all
else
{
self -> tlsg . allow_all_certs = allow_all_certs;
/*
* We are acting from supposition that at some particular
* moments there should be called initlialisation of
* TLS configurations, which will be reflected at next
* handshake
*/
if ( self -> tlsg . safe_to_modify_ssl_config ) {
vdb_mbedtls_ssl_conf_authmode(
&self -> tlsg . config,
( self -> tlsg . allow_all_certs
? MBEDTLS_SSL_VERIFY_OPTIONAL
: MBEDTLS_SSL_VERIFY_REQUIRED
)
);
}
}
return rc;
......
......@@ -2201,7 +2201,7 @@ uint64_t CC VTableCursorGetCacheCapacity(const VCURSOR_IMPL *self)
const PageMapProcessRequest* VTableCursorPageMapProcessRequest(const struct VCURSOR_IMPL *self)
{
assert ( self != NULL );
return self->pagemap_thread ? &self->pmpr : NULL;
return & self -> pmpr;
}
const struct VTable * VTableCursorGetTable ( const struct VCURSOR_IMPL * self )
......
......@@ -24,4 +24,4 @@
*
*/
#define LIBVDB_VERS 0x0207001B
#define LIBVDB_VERS 0x0207001C
......@@ -31,6 +31,7 @@ TOP ?= $(abspath ../..)
MODULE = test/kns
TEST_TOOLS = \
test200for-whole-file \
test-kns \
test-http-dropconn \
KNSManagerSingletonTest \
......@@ -126,6 +127,18 @@ $(TEST_BINDIR)/test-kns: $(KNSTEST_OBJ)
kns: test-kns
$(TEST_BINDIR)/test-kns #-l=all
#----------------------------------------------------------------
# test200for-whole-file
#
K2_SRC = \
test200for-whole-file \
K2_OBJ = \
$(addsuffix .$(OBJX),$(K2_SRC))
$(TEST_BINDIR)/test200for-whole-file: $(K2_OBJ)
$(LP) --exe -o $@ $^ $(KNSTEST_LIB)
#----------------------------------------------------------------
# test-proxy
#
......
/*===========================================================================
*
* PUBLIC DOMAIN NOTICE
* National Center for Biotechnology Information
*
* This software/database is a "United States Government Work" under the
* terms of the United States Copyright Act. It was written as part of
* the author's official duties as a United States Government employee and
* thus cannot be copyrighted. This software/database is freely available
* to the public for use. The National Library of Medicine and the U.S.
* Government have not placed any restriction on its use or reproduction.
*
* Although all reasonable efforts have been taken to ensure the accuracy
* and reliability of the software and data, the NLM and the U.S.
* Government do not and cannot warrant the performance or results that
* may be obtained by using this software or data. The NLM and the U.S.
* Government disclaim all warranties, express or implied, including
* warranties of performance, merchantability or fitness for any particular
* purpose.
*
* Please cite the author in any work or product based on this material.
*
* =========================================================================== */
#include <kapp/args.h> /* ArgsMakeAndHandle */
#include <kfg/kfg-priv.h> /* KConfigMakeEmpty */
#include <kfs/file.h> /* KFileRelease */
#include <klib/debug.h> /* KDbgSetString */
#include <kns/http.h> /* KNSManagerMakeHttpFile */
#include <kns/manager.h> /* KNSManagerRelease */
#include <ktst/unit_test.hpp> /* TEST_SUITE_WITH_ARGS_HANDLER */
#define RELEASE( type, obj ) do { rc_t rc2 = type##Release ( obj ); \
if (rc2 != 0 && rc == 0) { rc = rc2; } obj = NULL; } while ( false )
static rc_t argsHandler ( int argc, char * argv [] )
{ return ArgsMakeAndHandle ( NULL, argc, argv, 0, NULL, 0 ); }
TEST_SUITE_WITH_ARGS_HANDLER ( T200FOR_WHOLE_FILE, argsHandler )
TEST_CASE ( Test_206) {
KNSManager * mgr = NULL;
REQUIRE_RC ( KNSManagerMake ( & mgr ) );
const KFile * file = NULL;
REQUIRE_RC ( KNSManagerMakeHttpFile ( mgr, & file, NULL, 0x01010000,
"https://sra-download.ncbi.nlm.nih.gov/traces/refseq/KC702174.1" ) );
uint64_t size = 0;
REQUIRE_RC ( KFileSize ( file, & size ) );
void * buffer = malloc ( size );
REQUIRE_NOT_NULL ( buffer );
size_t num_read = 0;
// read incomplete file: expect 206 response code
size_t bsize = size - 1;
REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
REQUIRE_EQ ( num_read, bsize );
// read the whole: storage.googleapis.com returns 200
bsize = size;
REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
REQUIRE_EQ ( num_read, bsize );
// request more that file size: expect exact file size
REQUIRE_RC ( KFileRead ( file, 0, buffer, size * 2, & num_read ) );
REQUIRE_EQ ( num_read, static_cast < size_t > ( size ) );
free ( buffer );
REQUIRE_RC ( KFileRelease ( file ) );
REQUIRE_RC ( KNSManagerRelease ( mgr ) );
}
TEST_CASE ( Test_200 ) {
KNSManager * mgr = NULL;
REQUIRE_RC ( KNSManagerMake ( & mgr ) );
const KFile * file = NULL;
REQUIRE_RC ( KNSManagerMakeHttpFile ( mgr, & file, NULL, 0x01010000,
"https://storage.googleapis.com/yan-blastdb/2018-09-12-08-33-02/fuse.xml"
) );
uint64_t size = 0;
REQUIRE_RC ( KFileSize ( file, & size ) );
void * buffer = malloc ( size );
REQUIRE_NOT_NULL ( buffer );
size_t num_read = 0;
// read incomplete file: expect 206 response code
size_t bsize = size - 1;
REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
REQUIRE_EQ ( num_read, bsize );
// read the whole: storage.googleapis.com returns 200
bsize = size;
REQUIRE_RC ( KFileRead ( file, 0, buffer, bsize, & num_read ) );
REQUIRE_EQ ( num_read, bsize );
// request more that file size: expect exact file size
REQUIRE_RC ( KFileRead ( file, 0, buffer, size * 2, & num_read ) );
REQUIRE_EQ ( num_read, static_cast < size_t > ( size ) );
free ( buffer );
REQUIRE_RC ( KFileRelease ( file ) );
REQUIRE_RC ( KNSManagerRelease ( mgr ) );
}
extern "C" {
const char UsageDefaultName[] = "test200for-whole-file";
rc_t CC UsageSummary ( const char * progname) { return 0; }
rc_t CC Usage ( const struct Args * args ) { return 0; }
ver_t CC KAppVersion ( void ) { return 0; }
rc_t CC KMain ( int argc, char * argv [] ) { if (
0 ) assert ( ! KDbgSetString ( "KNS-HTTP" ) );
KConfigDisableUserSettings ();
KConfig * kfg = NULL;
rc_t rc = KConfigMakeEmpty ( & kfg );
// turn off certificate validation to download from storage.googleapis.com
if ( rc == 0 )
rc = KConfigWriteString ( kfg, "/tls/allow-all-certs", "true" );
if ( rc == 0 )
rc = T200FOR_WHOLE_FILE ( argc, argv );
RELEASE ( KConfig, kfg );
return rc;
}
}
......@@ -550,7 +550,9 @@ FIXTURE_TEST_CASE( VTableCursor_PageMapProcessRequest, TableCursorFixture )
// per non-cached cursor (see VTableCreateCachedCursorReadImpl() in cursor-table.c)
REQUIRE_RC ( VCursorLaunchPagemapThread ( (VCursor*)m_cur ) );
const PageMapProcessRequest * req = VCursorPageMapProcessRequest ( m_cur );
REQUIRE_NULL ( req );
// VDB-3652: VCursorPageMapProcessRequest returns a non-NULL value even if the thread is not running
REQUIRE_NOT_NULL ( req );
REQUIRE_NULL ( req -> pm );
}
FIXTURE_TEST_CASE( VTableCursor_CacheActive, TableCursorFixture )
......