CHANGES.md

 # NCBI External Developer Release:
 
+
+## SRA Toolkit 2.10.2
+**January 15, 2020**
+
+  **cloud, kfg, vdb-config**: added command line options for cloud configuration  
+  **fasterq-dump**: fixed bug of random error at startup  
+  **kfg, kns, krypto, prefetch, vdb-config, vfs**: names service URL was updated to locate.ncbi.nlm.nih.gov  
+  **kfg, prefetch, vfs**: fixed possibility to prefetch karts with genotype and phenotype files  
+  **latf-load**: latf-load now preserves read names  
+  **prefetch**: accepts JWT cart command line argument  
+  **prefetch**: accepts JWT cart command line argument plus accession filter  
+  **prefetch**: fixed crash when run with --output-file option  
+  **prefetch**: make sure to accept old style kart file; added --cart command line option  
+  **prefetch**: the download transport has been limited to https and the eliminate-quals option has been temporarily disabled  
+  **prefetch, vfs**: allow to specify file type to resolve  
+  **prefetch, vfs**: allow to use "prefetch --type all" to request download of all file types  
+  **sra-pileup**: printing bases inbetween slices when user-defined ref-name given fixed  
+  **sra-stat**: can accept any number of reads  
+  **srapath**: fixed regression when an extra vdbcache URL is printed  
+  **srapath**: fixed regression when resolving protected data  
+  **sratools**: add --perm option to all accessor tools  
+  **sratools**: added --ngc command line argument  
+  **sratools**: fastq-dump accepts both -v and --verbose to enable verbose mode  
+  **sratools**: sratools accept --ngc to specify the ngc file  
+  **sratools**: sratools accepts -Z for fasterq-dump  
+  **sratools**: sratools was rewritten as a stand-alone binary  
+  **sratools, vdb-config**: dbGaP page removed  
+  **vdb, vdb-config**: GUID shown in vdb-config or created if not yet present  
+  **vdb, vdb-dump**: fixed an error reporting bug  
+  **vdb-config**: allow multiple saves in interactive mode  
+  **vdb-config**: disabled possibility to import ngc file  
+  **vdb-config**: fixed bug in file-select-dialog  
+  **vdb-config**: wording of public user-repository changed to just user-repository  
+  **vdb-dump**: fixed bug in -X ( hex ) mode  
+  **vdb-dump, vfs**: addressed obscure bug preventing access to a single external reference  
+  **vdb-validate**: fixed handling of empty cells affecting certain databases  
+
+
+## SRA Toolkit 2.10.1
+**December 16, 2019**
+
+  **sra-tools**: changed version to match that of _ncbi-vdb_
+
+
 ## SRA Toolkit 2.10.0
 **August 19, 2019**
 

build/Makefile.env

@@ -356,8 +356,8 @@ DLLX ?= $(SHLX)
 ARCHDEFS = -D_ARCH_BITS=__SIZEOF_POINTER__*__CHAR_BIT__ -DLIBPREFIX=$(LPFX) -DSHLIBEXT=$(DLLX)
 
 # default tool parameters
-CFLAGS	= $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
-CPFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+CFLAGS	= -std=gnu11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+CPFLAGS = -std=gnu++11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
 LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS)
 
 #-------------------------------------------------------------------------------

build/Makefile.install

@@ -69,10 +69,6 @@ installexes:
 	@ echo "Installing executables to $(INST_BINDIR)/..."
 	@ mkdir -p $(INST_BINDIR)
 	@ $(MAKE) -s --no-print-directory -f $(TOP)/build/Makefile.install $(TOOLS_TO_INSTALL)
-	@ # copy the driver 
-	 $(COPY) $(TOP)/tools/driver-tool/sratools.pl $(INST_BINDIR)/sratools$(VERSION_EXEX)
-	 ln -f -s sratools$(VERSION_EXEX) $(INST_BINDIR)/sratools$(MAJVERS_EXEX)
-	 ln -f -s sratools$(MAJVERS_EXEX) $(INST_BINDIR)/sratools$(EXEX)
 	@ #
 	@ printf "Installing magic file..."
 	@ $(COPY) $(TOP)/tools/copycat/magic $(INST_BINDIR) && echo "done"
@@ -132,7 +128,7 @@ endif
 #
 
 TO_UNINSTALL = $(addsuffix *,$(addprefix $(INST_BINDIR)/,$(TOOLS_TO_INSTALL) \
-				magic sratools sratools.pl))
+				magic ))
 ifneq (true, $(LINUX_ROOT))
     TO_UNINSTALL += $(INST_BINDIR)/ncbi/certs.kfg $(INST_BINDIR)/ncbi/ncbi-vdb.kfg $(INST_BINDIR)/ncbi/default.kfg $(INST_BINDIR)/ncbi/vdb-copy.kfg
 endif

build/Makefile.shell

@@ -116,7 +116,7 @@ endif
 
 # determine SRCDIR
 ifdef MODULE
-    SRCDIR=$(TOP)/$(MODULE)
+    override SRCDIR=$(TOP)/$(MODULE)
 else
     SRCDIR=$(TOP)
 endif

build/Makefile.tools

@@ -17,12 +17,12 @@ TOOLS_TO_COPY = \
 	vdb-config      \
 	vdb-copy        \
 	vdb-decrypt     \
-	vdb-dump        \
 	vdb-encrypt     \
 	vdb-lock        \
 	vdb-passwd      \
 	vdb-unlock      \
 	vdb-validate    \
+	sratools        \
 
 # these tools will be copied to <instdir>/<toolname>-orig;
 # <toolname> will be created as a symlink to sratools
@@ -33,6 +33,7 @@ TOOLS_TO_SYMLINK = \
 	prefetch        \
 	srapath         \
 	sra-pileup      \
+	vdb-dump        \
 
 # these tools may be missing from $(BINDIR), not an error
 TOOLS_TO_COPY_OPTIONAL = \

build/Makefile.vers

@@ -23,4 +23,4 @@
 # ===========================================================================
 
 # SRA-TOOLS and library version
-VERSION = 2.10.0
+VERSION = 2.10.2

build/check-exit-code.py

@@ -48,7 +48,7 @@ if not os.access ( BANANA, os.X_OK ):
 
 def check_stream ( stream ):
     for line in stream:
-        if line.startswith ( "Usage:" ):
+        if line.startswith ( b"Usage:" ):
             return True
     return False
 
@@ -128,4 +128,4 @@ if ANANASA:
 ## C'onclusion
 ##
 
-print ( "[" + os.path.basename ( __file__ ) + "] test passed for [" + BANANA + "]\n" )
+print ( "[" + os.path.basename ( __file__ ) + "] test passed for [" + BANANA + "]" )

build/install-kfg.sh

@@ -38,13 +38,45 @@ TGT_FILE=$3/$1
 
 mkdir -p ${KONFIG_DIR}
 
+function get_md5 ()
+{
+    local MD5="$(which md5sum)"
+    if [ "$MD5" != "" ] && [ -x "$MD5" ]
+    then
+        "$MD5" "$1" | awk '{print $1;}'
+    else
+        MD5=/usr/bin/md5sum
+        if [ -x "$MD5" ]
+        then
+            "$MD5" "$1" | awk '{print $1;}'
+        else
+            MD5="$(which md5)"
+            if [ "$MD5" != "" ] && [ -x "$MD5" ]
+            then
+                "$MD5" -q "$1"
+            else
+                MD5=/sbin/md5
+                if [ -x "$MD5" ]
+                then
+                    "$MD5" -q "$1"
+                else
+                    echo "failed to locate md5 tool" 2>&1 1>/dev/null
+                    exit 5
+                fi
+            fi
+        fi
+    fi
+}
+
 #echo "installing $1 from $2 to $3, mdsums = $4"
 
 # create a backup if installed file has been modified by user
-if [ -f ${TGT_FILE} ] ; then
-    md5=$(md5sum ${TGT_FILE} | awk '{print $1;}')
+if [ -f ${TGT_FILE} ]
+then
+    md5=$(get_md5 ${TGT_FILE})
     #echo "$1 md5=$md5"
-    if [ "$(grep ${md5} ${MD5SUMS})" == "" ] ; then
+    if [ "$(grep ${md5} ${MD5SUMS})" == "" ]
+    then
         # not a known version of the file; create a backup copy
         mv -b -v ${TGT_FILE} ${TGT_FILE}.orig
     fi

configure

@@ -41,5 +41,5 @@ else
         echo >&2 "configure: error: perl not found."; exit 1; }
 
     cd $CURDIR/setup
-    perl -w konfigure.perl "$@"
+    perl konfigure.perl "$@"
 fi

debian/changelog

-sra-sdk (2.10.0+dfsg-1) UNRELEASED; urgency=medium
+sra-sdk (2.10.2+dfsg-1) UNRELEASED; urgency=medium
 
   * Team upload.
   * New upstream version
   * debhelper-compat 12
   * Standards-Version: 4.4.1
 
-  * TODO: Fails to build from source since it does not find
-          KClientHttpRequestFormatPostMsg to link against
+  * ncbi-vdb needs to appear as 2.10.2 in Debian with its mbedtls wrapper
+    fixed to avoid a link error in this package.
 
  -- Steffen Moeller <moeller@debian.org>  Thu, 28 Nov 2019 21:39:13 +0100
 

debian/patches/drop_tests.patch

@@ -6,7 +6,7 @@ Index: sra-sdk/test/vdb-config/Makefile
 ===================================================================
 --- sra-sdk.orig/test/vdb-config/Makefile
 +++ sra-sdk/test/vdb-config/Makefile
-@@ -47,8 +47,9 @@ runtests: test-vdb-config
+@@ -88,8 +88,9 @@ runtests: test-vdb-config model
  
  test-vdb-config:
  	@ printf "Testing exit code vdb-config of vdb-config... "

debian/patches/hardening.patch

@@ -6,8 +6,10 @@ Description: Propagate Debian hardening options into build system
   setting inside the sequence of Makefiles which leads to a loss of
   DEFINES and INCDIRS which are needed to build properly
 
---- a/Makefile
-+++ b/Makefile
+Index: sra-sdk/Makefile
+===================================================================
+--- sra-sdk.orig/Makefile
++++ sra-sdk/Makefile
 @@ -155,11 +155,11 @@ $(RHOSTS):
  	@ $(MAKE) -s TOP=$(CURDIR) -f build/Makefile.env rebuild-dirlinks config
  
@@ -23,16 +25,19 @@ Description: Propagate Debian hardening options into build system
  
  .PHONY: $(PASSTHRUS) $(RHOSTS) $(REPORTS)
  
---- a/build/Makefile.env
-+++ b/build/Makefile.env
+Index: sra-sdk/build/Makefile.env
+===================================================================
+--- sra-sdk.orig/build/Makefile.env
++++ sra-sdk/build/Makefile.env
 @@ -356,9 +356,9 @@ DLLX ?= $(SHLX)
  ARCHDEFS = -D_ARCH_BITS=__SIZEOF_POINTER__*__CHAR_BIT__ -DLIBPREFIX=$(LPFX) -DSHLIBEXT=$(DLLX)
  
  # default tool parameters
--CFLAGS	= $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
-+CFLAGS	= $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS) $(DEB_CFLAGS)
- CPFLAGS = $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+-CFLAGS	= -std=gnu11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
+-CPFLAGS = -std=gnu++11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS)
 -LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS)
++CFLAGS	= -std=gnu11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(PED) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS) $(DEB_CFLAGS)
++CPFLAGS = -std=gnu++11 $(DEBUG) $(DBG) $(CARCH) $(PROF) $(DEFINES) $(ARCHDEFS) $(MIN_DEPLOY_OS_OPT) $(INCDIRS) $(DEB_CXXFLAGS)
 +LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS) $(DEB_LDFLAGS)
  
  #-------------------------------------------------------------------------------

debian/patches/ignore_failing_test_for_the_moment_FIXME.patch

@@ -17,7 +17,7 @@ Index: sra-sdk/test/Makefile
 ===================================================================
 --- sra-sdk.orig/test/Makefile
 +++ sra-sdk/test/Makefile
-@@ -36,23 +36,16 @@ include $(TOP)/build/Makefile.config
+@@ -36,20 +36,13 @@ include $(TOP)/build/Makefile.config
  #
  SUBDIRS =    \
  	read-filter-redact \
@@ -25,11 +25,8 @@ Index: sra-sdk/test/Makefile
  	vdb-config      \
  	align-info      \
 -	align-cache     \
- 	vdb-validate    \
  	kar             \
 -	copycat         \
- 	vdb-copy        \
- 	qual-recalib-stat \
 -	sra-stat        \
 -	srapath         \
  	sra-sort        \
@@ -38,9 +35,9 @@ Index: sra-sdk/test/Makefile
  	fastq-loader    \
 -	kget            \
 -	vdb-dump        \
- 
- # under construction
- #    ngs-pileup      \
+ 	vdb-copy        \
+ 	qual-recalib-stat \
+ 	vdb-validate      \
 @@ -62,6 +55,8 @@ include $(TOP)/build/Makefile.targets
  
  $(SUBDIRS):

debian/patches/ignore_failing_tests_for_the_moment.patch

@@ -72,4 +72,4 @@ Index: sra-sdk/test/Makefile
 -	vcf-loader      \
  	align-info      \
  	align-cache     \
- 	vdb-validate    \
+ 	kar             \

debian/rules

@@ -5,6 +5,9 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 
 pkg := sra-toolkit
 
+export CFLAGS += -I/usr/include/ncbi-vdb
+export CXXFLAGS += -I/usr/include/ncbi-vdb
+
 %:
 	dh $@ --no-parallel
 
@@ -16,6 +19,8 @@ override_dh_auto_clean:
 override_dh_clean:
 	dh_clean
 	rm -f build/OUTDIR.linux
+	rm -f reconfigure build/ld.linux.exe_cmd.sh build/Makefile.config build/Makefile.config.linux.x86_64
+
 
 override_dh_update_autotools_config:
 	echo "do nothing"
@@ -33,7 +38,7 @@ override_dh_auto_configure:
 	./configure --build=$(MULTIARCH) --prefix=$(CURDIR)/debian/tmp/usr --build-prefix=$(CURDIR)/debian/tmp/usr --with-ngs-sdk-prefix=/usr
 
 override_dh_auto_build:
-	export LD_LIBRARY_PATH="$$LD_LIBRARY_PATH:$(OUTDIR)/lib"; export OUTDIR=$(OUTDIR) ; make DEB_CFLAGS="$(CFLAGS)" DEB_LDFLAGS="$(LDFLAGS)"
+	export LD_LIBRARY_PATH="$$LD_LIBRARY_PATH:$(OUTDIR)/lib"; export OUTDIR=$(OUTDIR) ; make DEB_CFLAGS="$(CFLAGS)" DEB_CXXFLAGS="$(CXXFLAGS)" DEB_LDFLAGS="$(LDFLAGS)" 
 
 override_dh_install:
 	# upstream installs versioned *.a libs - we only want a single version

shared/toolkit.vers

-2.10.0
+2.10.2

shared/toolkit.vers.h

-#define TOOLKIT_VERS 0x020A0000
+#define TOOLKIT_VERS 0x020A0002

test/Makefile

@@ -42,11 +42,8 @@ SUBDIRS =    \
 	vcf-loader      \
 	align-info      \
 	align-cache     \
-	vdb-validate    \
 	kar             \
 	copycat         \
-	vdb-copy        \
-	qual-recalib-stat \
 	sra-stat        \
 	srapath         \
 	sra-sort        \
@@ -55,6 +52,9 @@ SUBDIRS =    \
 	fastq-loader    \
 	kget            \
 	vdb-dump        \
+	vdb-copy        \
+	qual-recalib-stat \
+	vdb-validate      \
 
 # under construction
 #    ngs-pileup      \

test/install/Makefile

@@ -81,6 +81,8 @@ test-install: cleanup
 	@ for i in $(INSTALLED_FILES); do test -f "$$i" || ( echo "$$i is not found" >>$(INSTALL_DIR)/missing ); done
 	@ ! test -f $(INSTALL_DIR)/missing || ! cat $(INSTALL_DIR)/missing
 	@ #
+	@ # make sure sratools is not Perl
+	( ! file $(INST_BINDIR)/sratools$(VERSION_EXEX) | grep Perl ) || ( echo "Error: $(INST_BINDIR)/sratools is a Perl script" && exit 1 )
 	@ #
 	@ echo "2. uninstall"
 	@ #

test/kar/Makefile

@@ -40,16 +40,9 @@ runtests: kar_test
 #-------------------------------------------------------------------------------
 # scripted tests
 #
-kar_test: kar-test.sh
+kar_test: kar-ntest.sh
 	@ echo "Starting kar tests..."
-	@ bash kar-test.sh $(BINDIR)/kar
-
-kar:
-	rm -rf actual
-	mkdir -p actual
-	$(BINDIR)/kar -c actual/1.0.kar -d input/1.0; $(BINDIR)/kar -t actual/1.0.kar >actual/1.0.stdout; diff expected/1.0.stdout actual/1.0.stdout
-	rm -rf actual
-	@ python $(TOP)/build/check-exit-code.py $(BINDIR)/kar
+	@ bash kar-ntest.sh $(BINDIR)/kar
 
 .PHONY: $(TEST_TOOLS)