Commit 4ca93d1a authored by Andreas Tille's avatar Andreas Tille

New upstream version 2.9.0+dfsg

parent f9830f55
......@@ -21,3 +21,4 @@
/.cproject
/.project
/.pydevproject
tmp.kfg
\ No newline at end of file
# NCBI External Developer Release:
## SRA Toolkit 2.9.0
**February 23, 2018**
**bam-load**: an issue with accessing WGS accessions was fixed
**bam-load**: bam-load will generate an error and quit when too many spots have been encountered
**bam-load**: renamed an internal function to avoid a name conflict
**bam-load, fastq-load**: updated to use better thread termination signaling
**bam-load, sra-stat**: Updated sra-stat to extract statistics of alterations made by loaders for inclusion in its report
**build**: Created a script that allows to add a new volume to existing repository
**build**: Fixed configure allowing to run it on Perl with version >= v5.26 that has "." removed from @INC
**build**: added "smoke tests"
**build**: recognize version of libhdf5 that does not allow static linking and do not try to use it
**build, doc**: added wiki page: Building-from-source-:--configure-options-explained
**build, ncbi-vdb, sra-tools**: the installation script now saves configuration files if they were modified by the user
**build, sra-tools**: "make runtests" now invokes "make all"
**build, vdb-sql**: modified build to avoid vdb-sql in absence of libxml2
**fastq-dump**: minor change to help text
**fastq-dump, vdb**: Fixed crashing of fastq-dump when dumping multiple runs with -split-3 option specified
**fastq-load**: preserves spot names when the platform is Illumina
**kfg**: added searching of configuration files in ../etc/ncbi/ relative to the binaries
**kfg, prefetch**: set limit of Aspera usage to 450m
**kfg, prefetch, remote-fuser, vfs**: Updated resolving of cache location of non-accession objects
**klib**: Reverted KTimeMakeTime to use UTC
**kns**: Accept the same http_proxy specifications as wget
**kns**: Added possibility to report server's IP address after network error
**kns**: Ignore HTTP headers sent multiple times
**kns**: Improved reporting of network errors
**kns**: fixed generation of invalid error code in response to dropped connection
**latf-load**: now processing multi-line sequences and qualities
**latf-load**: pacbio spot names with a range are now processed correctly
**pileup-stats**: pileup-stats now exits with code 3 if called without arguments
**prefetch**: fixed a bug in prefech when it printed invalid error messages after failure of reading from server
**sra-search**: added option --fasta for output in FASTA format
**sra-search**: added option to display version number
**sra-search**: added option to search unaligned and partially aligned fragments only
**sra-search**: improved performance in reference-driven mode
**sra-search**: various efficiency/readability improvements in the code
**sra-sort**: Created a separate version of sra-sort for Complete Genomics
**sra-sort**: Fixed race condition in sra-sort when result was not completed when using fast drives
**sra-stat**: Added calculation of N50, L50, N90, L90 statistics
**sra-stat**: Fixed: sra-stat prints the path of alignment reference when the path is remote (http)
**sra-stat**: Improved performance when calculating bases statistics
**sra-stat**: The maximum number of reads that can be processed was Increased to 4K.
**sra-tools, vdb**: access to vdb/ngs via SQLite
**srapath**: srapath allows to get results of name resolver CGI
**vdb-config**: vdb-config does not fail when /repository/user/default-path is not set in configuration
**vdb-validate**: added a check of sum(READ_LEN) against length(READ)
**vfs**: Name resolving service was updated and switched to protocol version 3.0
## SRA Toolkit 2.8.2
**March 6, 2017**
......@@ -191,5 +241,3 @@
**var-expand**: a tool for batch variation expansion
**vdb-config**: now handles standard options such as --option-file
**vdb-validate**: Added code to continue with the next row when column has discontiguous blobs
......@@ -200,7 +200,7 @@ endif
.PHONY: stdclean removelinks makedirs vers-includes rebuild-dirlinks
.PHONY: stdjclean makejdirs
# configuration targets
# configuration targets
out:
@ echo $(OUTDIR) > $(TOP)/build/OUTDIR.$(BUILD_OS)
@ $(MAKE) TOP=$(TOP) -f $(TOP)/build/Makefile.env rebuild-dirlinks
......@@ -362,11 +362,11 @@ LDFLAGS = $(DBG) $(PROF) $(CARCH) $(MIN_DEPLOY_OS_OPT) $(LDPATHS)
#-------------------------------------------------------------------------------
# runtests
#
#
# MallocScribble=1 is for catching allocation problems on Mac
#
ifeq ($(RUNTESTS_OVERRIDE),)
runtests: std $(TEST_TOOLS)
runtests: all $(TEST_TOOLS)
@ export VDB_CONFIG=$(VDB_CONFIG);export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;export MallocScribble=1;\
for i in $(TEST_TOOLS);\
do\
......@@ -380,11 +380,11 @@ endif
#-------------------------------------------------------------------------------
# slowtests
#
#
# $(SLOWTESTSDATADIR) should be used to create temporary test files
SLOWTESTSDATADIR ?= /panfs/pan1.be-md.ncbi.nlm.nih.gov/sra-test/slowtests/$(shell whoami)
slowtests: std $(SLOW_TEST_TOOLS)
slowtests: all $(SLOW_TEST_TOOLS)
@ export LD_LIBRARY_PATH=$(LIBDIR):$$LD_LIBRARY_PATH;\
for i in $(SLOW_TEST_TOOLS);\
do\
......
......@@ -44,13 +44,25 @@ endif
#fake root for debugging
#uncomment this line and change the test for root ( see under install: ) to succeed:
#ROOT = ~/root
ifeq (linux, $(OS))
ifeq (0, $(shell id -u))
LINUX_ROOT = true
endif
endif
ifeq (true, $(LINUX_ROOT))
KONFIG_DIR = $(ROOT)/etc/ncbi
PROFILE_FILE = $(ROOT)/etc/profile.d/ncbi-vdb
else
KONFIG_DIR = $(INST_BINDIR)/ncbi
endif
EXE_TARGET = $(INST_BINDIR)
PROFILE_FILE = $(ROOT)/etc/profile.d/sra-tools
installexes:
installexes:
@ echo "Installing executables to $(INST_BINDIR)/..."
@ mkdir -p $(INST_BINDIR)
@ mkdir -p $(INST_BINDIR)
@ $(MAKE) -s --no-print-directory -f $(TOP)/build/Makefile.install $(TOOLS_TO_INSTALL)
@ printf "Installing magic file..."
@ $(COPY) $(TOP)/tools/copycat/magic $(INST_BINDIR) && echo "done"
......@@ -61,32 +73,18 @@ $(TOOLS_TO_INSTALL):
@ - ( test -f $(BINDIR)/$@$(EXEX) || ( echo "not found" && false ) ) && \
$(COPY) $(BINDIR)/$@$(EXEX) $(BINDIR)/$@$(VERSION_EXEX) $(BINDIR)/$@$(MAJVERS_EXEX) $(INST_BINDIR) && \
echo "done"
ifeq (linux, $(OS))
ifeq (0, $(shell id -u))
LINUX_ROOT = true
endif
endif
install: installexes
copykfg:
@ echo "Installing configuration files to $(KONFIG_DIR)"
@ $(TOP)/build/install-kfg.sh default.kfg $(BINDIR)/ncbi $(KONFIG_DIR) $(TOP)/shared/kfgsums
@ $(TOP)/build/install-kfg.sh certs.kfg $(BINDIR)/ncbi $(KONFIG_DIR) $(TOP)/shared/kfgsums
@ $(TOP)/build/install-kfg.sh vdb-copy.kfg $(TOP)/tools/vdb-copy $(KONFIG_DIR) $(TOP)/shared/kfgsums
@ # rename obsolete ncbi-vdb.kfg
@ if [ -f $(KONFIG_DIR)/ncbi-vdb.kfg ] ; \
then mv -v $(KONFIG_DIR)/ncbi-vdb.kfg $(KONFIG_DIR)/ncbi-vdb.kfg.orig ; fi
install: installexes copykfg
ifeq (true, $(LINUX_ROOT))
@ # install configuration file(s)
@ printf "Installing configuration files to $(ROOT)/etc/ncbi/..."
( mkdir -p $(ROOT)/etc/ncbi && \
cp -p $(TOP)/tools/vdb-copy/vdb-copy.kfg $(ROOT)/etc/ncbi ; \
if [ -f $(BINDIR)/ncbi/default.kfg ] ; then \
if [ -f $(ROOT)/etc/ncbi/default.kfg ] ; then \
mv -v $(ROOT)/etc/ncbi/default.kfg $(ROOT)/etc/ncbi/default.kfg.orig;\
fi; \
cp -p $(BINDIR)/ncbi/default.kfg $(ROOT)/etc/ncbi ; \
fi; \
if [ -f $(BINDIR)/ncbi/certs.kfg ] ; then \
if [ -f $(ROOT)/etc/ncbi/certs.kfg ] ; then \
mv -v $(ROOT)/etc/ncbi/certs.kfg $(ROOT)/etc/ncbi/certs.kfg.orig; \
fi; \
cp -p $(BINDIR)/ncbi/certs.kfg $(ROOT)/etc/ncbi ; \
fi ) && echo done
@ #
@ echo "Updating $(PROFILE_FILE).[c]sh"
@ printf \
"#version $(VERSION)\n"\
......@@ -100,30 +98,11 @@ ifeq (true, $(LINUX_ROOT))
"if ( \$$status ) setenv PATH $(EXE_TARGET):\$$PATH\n"\
>$(PROFILE_FILE).csh && chmod 644 $(PROFILE_FILE).csh || true;
@ #TODO: check version of the files above
else
@ # install configuration file(s)
@ printf "Installing configuration files to $(INST_BINDIR)/ncbi/..."
( mkdir -p $(INST_BINDIR)/ncbi && \
cp -p $(TOP)/tools/vdb-copy/vdb-copy.kfg $(INST_BINDIR)/ncbi/ ; \
if [ -f $(BINDIR)/ncbi/default.kfg ] ; then \
if [ -f $(INST_BINDIR)/ncbi/default.kfg ] ; then \
mv -v $(INST_BINDIR)/ncbi/default.kfg \
$(INST_BINDIR)/ncbi/default.kfg.orig; \
fi; \
cp -p $(BINDIR)/ncbi/default.kfg $(INST_BINDIR)/ncbi ; \
fi; \
if [ -f $(BINDIR)/ncbi/certs.kfg ] ; then \
if [ -f $(INST_BINDIR)/ncbi/certs.kfg ] ; then \
mv -v $(INST_BINDIR)/ncbi/certs.kfg \
$(INST_BINDIR)/ncbi/certs.kfg.orig; \
fi; \
cp -p $(BINDIR)/ncbi/certs.kfg $(INST_BINDIR)/ncbi ; \
fi ) && echo done
endif
#-------------------------------------------------------------------------------
# uninstall
#
#
TO_UNINSTALL = $(addsuffix *,$(addprefix $(INST_BINDIR)/,$(TOOLS_TO_INSTALL) magic))
ifneq (true, $(LINUX_ROOT))
......
......@@ -23,4 +23,4 @@
# ===========================================================================
# SRA-TOOLS and library version
VERSION = 2.8.2
VERSION = 2.9.0
......@@ -22,87 +22,33 @@
# Please cite the author in any work or product based on this material.
#
# ===========================================================================
#echo "$0 $*"
# $1 - path to sra-tools executables (general-loader, vdb-dump)
# $2 - name of the dumper executable (vdb-dump, kdbmeta etc.)
# $3 - work directory (expected results under expected/, actual results and temporaries created under actual/)
# $4 - test case ID (expect a file input/$3.gl to exist)
# $5 - expected return code
# $6 - command line options for general-loader
# $7 - command line options for dumper
# install-kfg.sh
# copies file $1 from $2 to $3.
# Will create a backup copy if the file's md5 is not found in $4 (assumed to be a user's edit)
#
# return codes:
# 0 - passed
# 1 - could not create temp dir
# 2 - unexpected return code from general-loader
# 3 - vdb-dump failed on the output of general-loader
# 4 - outputs differ
BINDIR=$1
DUMPER=$2
WORKDIR=$3
CASEID=$4
RC=$5
LOAD_OPTIONS=$6
DUMP_OPTIONS=$7
FILE_NAME=$1
SRC_DIR=$2
KONFIG_DIR=$3
MD5SUMS=$4
DUMP="$BINDIR/$DUMPER"
LOAD="$BINDIR/general-loader"
TEMPDIR=$WORKDIR/actual/$CASEID
SRC_FILE=$2/$1
TGT_FILE=$3/$1
echo "running test case $CASEID"
mkdir -p ${KONFIG_DIR}
mkdir -p $TEMPDIR
if [ "$?" != "0" ] ; then
echo "cannot create "
exit 1
fi
rm -rf $TEMPDIR/*
CMD="cat input/$CASEID.gl | $LOAD $LOAD_OPTIONS 1>$TEMPDIR/load.stdout 2>$TEMPDIR/load.stderr"
#echo $CMD
eval $CMD
rc="$?"
if [ "$rc" != "$RC" ] ; then
echo "$LOAD returned $rc, expected $RC"
echo "command executed:"
echo $CMD
cat $TEMPDIR/load.stderr
exit 2
fi
#echo "installing $1 from $2 to $3, mdsums = $4"
if [ "$rc" == "0" ] ; then
CMD="$DUMP $TEMPDIR/db $DUMP_OPTIONS 1>$TEMPDIR/dump.stdout 2>$TEMPDIR/dump.stderr"
#echo $CMD
eval $CMD || ( echo "$CMD" && exit 3 )
# remove timestamps, date from metadata
sed -i -e 's/<timestamp>.*<\/timestamp>/<timestamp\/>/g' $TEMPDIR/dump.stdout
sed -i -e 's/date=".*" name/date="" name/g' $TEMPDIR/dump.stdout
diff $WORKDIR/expected/$CASEID.stdout $TEMPDIR/dump.stdout >$TEMPDIR/diff
rc="$?"
else
# remove timestamps
sed -i -e 's/^....-..-..T..:..:.. //g' $TEMPDIR/load.stderr
# remove pathnames
sed -i -e 's=/.*/==g' $TEMPDIR/load.stderr
# remove source locations
sed -i -e 's=: .*:[0-9]*:[^ ]*:=:=g' $TEMPDIR/load.stderr
# remove version number
sed -i -e 's=general-loader\(\.[0-9]*\)*=general-loader=g' $TEMPDIR/load.stderr
diff $WORKDIR/expected/$CASEID.stderr $TEMPDIR/load.stderr >$TEMPDIR/diff
rc="$?"
# create a backup if installed file has been modified by user
if [ -f ${TGT_FILE} ] ; then
md5=$(md5sum ${TGT_FILE} | awk '{print $1;}')
#echo "$1 md5=$md5"
if [ "$(grep ${md5} ${MD5SUMS})" == "" ] ; then
# not a known version of the file; create a backup copy
mv -b -v ${TGT_FILE} ${TGT_FILE}.orig
fi
fi
if [ "$rc" != "0" ] ; then
cat $TEMPDIR/diff
echo "command executed:"
echo $CMD
exit 4
fi
rm -rf $TEMPDIR
exit 0
# copy to the install location
cp ${SRC_FILE} ${TGT_FILE}
......@@ -41,5 +41,5 @@ else
echo >&2 "configure: error: perl not found."; exit 1; }
cd $CURDIR/setup
perl -w konfigure.perl "$@"
perl konfigure.perl "$@"
fi
This diff is collapsed.
......@@ -24,6 +24,10 @@
use strict;
use Cwd 'abs_path';
use File::Basename 'dirname';
use lib dirname( abs_path $0 );
sub println { print @_; print "\n" }
my ($filename, $directories, $suffix) = fileparse($0);
......@@ -433,6 +437,15 @@ if ($TOOLS =~ /gcc$/) {
$STATIC_LIBSTDCPP = check_static_libstdcpp();
}
if ( $PKG{REQ} ) {
foreach ( @{ $PKG{REQ} } ) {
unless (check_tool__h($_)) {
println "configure: error: '$_' cannot be found";
exit 1;
}
}
}
my @dependencies;
my %DEPEND_OPTIONS;
......@@ -656,6 +669,9 @@ foreach my $href (@REQ) {
} elsif ($quasi_optional && $found_itf && ($need_lib && ! $found_lib)) {
println "configure: $a{name} package: "
. "found interface files but not libraries.";
$found_itf = abs_path($found_itf);
push(@dependencies, "$a{aname}_INCDIR = $found_itf");
println "includes: $found_itf";
} else {
if ($OPT{'debug'}) {
$_ = "$a{name}: includes: ";
......@@ -1506,7 +1522,7 @@ sub check_compiler {
$flags = $n;
$log = ' int main() { }\n'
} elsif ($n eq 'hdf5') {
$library = '-lhdf5';
$library = '-Wl,-Bstatic -lhdf5 -Wl,-Bdynamic -ldl -lm -lz';
$log = '#include <hdf5.h> \n int main() { H5close (); }\n'
} elsif ($n eq 'fuse') {
$flags = '-D_FILE_OFFSET_BITS=64';
......@@ -1549,7 +1565,14 @@ sub check_compiler {
push ( @l, '' ) unless ( @l );
for my $i ( 0 .. $#l ) {
my $l = $l [ $i ];
next if ( $l && ! -d $l );
if ( $l && ! -d $l ) {
if ( $i == $#l ) {
println 'no';
return;
} else {
next;
}
}
my $gcc = "| $tool -xc $flags " . ($I ? "-I$I " : ' ')
. ($l ? "-L$l " : ' ') . "- $library";
$gcc .= ' 2> /dev/null' unless ($OPT{'debug'});
......@@ -1728,9 +1751,12 @@ EndText
foreach my $href (@REQ) {
next unless (optional($href->{type}));
my %a = %$href;
if ($a{option} =~ /-sources$/) {
if ($a{option} && $a{option} =~ /-sources$/) {
println " --$a{option}=DIR search for $a{name} package";
println " source files in DIR";
} elsif ($a{boption} && $a{boption} =~ /-build$/) {
println " --$a{boption}=DIR search for $a{name} package";
println " build output in DIR";
} else {
println " --$a{option}=DIR search for $a{name} files in DIR"
}
......
default.kfg 3b30b74826c463e41a78db9fbea972b2 2.7.0
default.kfg 5765f7186f138e30f985fecfd769d0ed 2.8.2
certs.kfg 94e16bb39bd24d526497b68bbe3b190f 2.8.2
vdb-copy.kfg 0e0b71eb7d3b496bacfb5c6d0bdd2806 2.8.2
#define TOOLKIT_VERS 0x02080002
#define TOOLKIT_VERS 0x02090000
......@@ -35,24 +35,25 @@ include $(TOP)/build/Makefile.config
# default
#
SUBDIRS = \
dump-test \
vdb-config \
fastq-loader \
vcf-loader \
kget \
general-loader \
vschema \
align-info \
align-cache \
vdb-dump \
vdb-validate \
kar \
copycat \
fastdump \
vdb-copy \
qual-recalib-stat \
sra-pileup \
sra-stat \
srapath \
sra-sort \
fastdump \
sra-pileup \
fuse \
fastq-loader \
kget \
vdb-dump \
# under construction
# ngs-pileup \
......@@ -65,3 +66,8 @@ $(SUBDIRS):
.PHONY: default $(SUBDIRS)
# time of slowtests in linux release mode
# real 4m 0.076s user 4m22.256s sys 0m10.707s sra-sort
# real 5m21.151s user 5m59.396s sys 0m39.296s fastdump
# real 6m55.921s user 6m28.035s sys 6m58.052s sra-pileup
# real 184m21.347s user 42m19.318s sys 259m56.028s fuse
# ===========================================================================
#
# PUBLIC DOMAIN NOTICE
# National Center for Biotechnology Information
#
# This software/database is a "United States Government Work" under the
# terms of the United States Copyright Act. It was written as part of
# the author's official duties as a United States Government employee and
# thus cannot be copyrighted. This software/database is freely available
# to the public for use. The National Library of Medicine and the U.S.
# Government have not placed any restriction on its use or reproduction.
#
# Although all reasonable efforts have been taken to ensure the accuracy
# and reliability of the software and data, the NLM and the U.S.
# Government do not and cannot warrant the performance or results that
# may be obtained by using this software or data. The NLM and the U.S.
# Government disclaim all warranties, express or implied, including
# warranties of performance, merchantability or fitness for any particular
# purpose.
#
# Please cite the author in any work or product based on this material.
#
# ==============================================================================
default: runtests
TOP ?= $(abspath ../..)
include $(TOP)/build/Makefile.env
runtests: check_fastq_dump_crash
check_fastq_dump_crash:
$(BINDIR)/fastq-dump -Z -split-3 ERR034677 SRR125365 > /dev/null
......@@ -37,16 +37,16 @@ include $(TOP)/build/Makefile.env
slowtests: fastdump1 fastdump2
ACC = SRR341578
SCRATCH = /tmp/$(shell whoami)/
SCRATCH ?= /tmp/$(shell whoami)
THREADS = 4
fastdump1: prepare $(BINDIR)/fastdump $(BINDIR)/vdb-dump
@ mkdir -p $(SCRATCH)
@./test_no_1.sh $(ACC) $(SCRATCH) $(THREADS) $(BINDIR)
@./test_no_1.sh $(ACC) $(SCRATCH)/ $(THREADS) $(BINDIR)
fastdump2: $(BINDIR)/fastdump $(BINDIR)/vdb-dump
@ mkdir -p $(SCRATCH)
@./test_no_2.sh $(ACC) $(SCRATCH) $(THREADS) $(BINDIR)
@./test_no_2.sh $(ACC) $(SCRATCH)/ $(THREADS) $(BINDIR)
prepare:
@ export BINDIR=$(BINDIR) ; export VDB_INCDIR=$(VDB_INCDIR) ; \
......
r
bt
up 5
p *seq
p read
l
do
l
up
l
p seq
p read
quit
r
bt
quit
b yy_get_next_buffer
r
c
r
disa 1
c
bt
l
up
p number_to_move
p yyextra->input
b 1708
r
c
c
p YY_CURRENT_BUFFER_LVALUE
p yyextra
p yyg->yyextra_r
p * yyg->yyextra_r
p * yyg->yyextra_r->input
c
bt
disa 2
r
up
p * yyg->yyextra_r->input
p * yyg->yyextra_r
p * yyg->yyextra_r->column
p FASTQ_input
quit
r
bt
up
up
up
p *pb
do
p yyscanner
p *yyscanner
do
p yyg
p *yyg
p *yyg->yyin_r
p yyg->yyin_r
p yyg->yyextra_r
p *yyg->yyextra_r
quit
quit
b wb-test-fastq.cpp:279
p pb
r
p pb
s
n
apropos watch
help watch
watch -l pb.input
c
up
p yyg->yyextra_r
p *yyg->yyextra_r
bt
r
disa 2
l
up
up
up
bt
bt
r
p pb
p &pb
bt
c
up
p (FASTQParseBlock *) 0x7fffffffdf58
p *(FASTQParseBlock *) 0x7fffffffdf58
l
p yyg->yyextra_r
quit
b wb-test-fastq.cpp:722
r
s
n
s
n
p *self
p length
p start
p stop
p start >= length || stop >= length || start >= stop
p start >= length
p stop >= length
p start >= stop
p start
p stop
quit
b wb-test-fastq-parse.cpp:321
r
p pb.record->seq
b ExpandRead
r
p copyTo
n
p copyTo
p *newReadLine
n
p pb -> readLength
p newReadLine -> tokenLength
n
p *pb
b FASTQ_input
c
p pb
p pb.record.seq
b fastq-reader.c:575
r
c
c
r
c
p ( const char *) ( pb . record -> source . base )
quit
b ExpandQuality