Skip to content
Commits on Source (4)
libmaus2 (2.0.657-1) unstable; urgency=medium
* allow limiting parallelism in fragment merging of BlockSortControl (implicitely reduces number of open files)
-- German Tischler-Höhle <germant@miltenyibiotec.de> Mon, 19 Aug 2019 14:29:09 +0200
libmaus2 (2.0.656-1) unstable; urgency=medium
* refactoring socket classes
* refactor and guard against interupted system calls in Socket.hpp
-- German Tischler-Höhle <germant@miltenyibiotec.de> Fri, 16 Aug 2019 10:54:29 +0200
libmaus2 (2.0.655-1) unstable; urgency=medium
* refactor AutoArray and avoid mismatching alloc/dealloc pairs (do not deallocate memory allocated through new [] using free)
-- German Tischler-Höhle <germant@miltenyibiotec.de> Thu, 15 Aug 2019 21:20:31 +0200
libmaus2 (2.0.654-1) unstable; urgency=medium
* remove extraneous include directive in SkewSuffixSort
* add missing include directive in TempFileArray
-- German Tischler-Höhle <germant@miltenyibiotec.de> Tue, 13 Aug 2019 15:58:13 +0200
libmaus2 (2.0.653-1) unstable; urgency=medium
* add getNextReferenceLength/getNextFrontDel/getNextReferenceInterval in BamAlignmentDecoderBase/BamAlignment
......
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.69 for libmaus2 2.0.652.
@%:@ Generated by GNU Autoconf 2.69 for libmaus2 2.0.656.
@%:@
@%:@ Report bugs to <tischler@mpi-cbg.de>.
@%:@
......@@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='libmaus2'
PACKAGE_TARNAME='libmaus2'
PACKAGE_VERSION='2.0.652'
PACKAGE_STRING='libmaus2 2.0.652'
PACKAGE_VERSION='2.0.656'
PACKAGE_STRING='libmaus2 2.0.656'
PACKAGE_BUGREPORT='tischler@mpi-cbg.de'
PACKAGE_URL='https://github.com/gt1/libmaus2'
 
......@@ -1609,7 +1609,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures libmaus2 2.0.652 to adapt to many kinds of systems.
\`configure' configures libmaus2 2.0.656 to adapt to many kinds of systems.
 
Usage: $0 [OPTION]... [VAR=VALUE]...
 
......@@ -1681,7 +1681,7 @@ fi
 
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of libmaus2 2.0.652:";;
short | recursive ) echo "Configuration of libmaus2 2.0.656:";;
esac
cat <<\_ACEOF
 
......@@ -1851,7 +1851,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
libmaus2 configure 2.0.652
libmaus2 configure 2.0.656
generated by GNU Autoconf 2.69
 
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -2724,7 +2724,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
 
It was created by libmaus2 $as_me 2.0.652, which was
It was created by libmaus2 $as_me 2.0.656, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
$ $0 $@
......@@ -3072,7 +3072,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
LIBRARY_VERSION=2:652:0
LIBRARY_VERSION=2:656:0
{ $as_echo "$as_me:${as_lineno-$LINENO}: Configuring for source in directory ${srcdir}" >&5
$as_echo "$as_me: Configuring for source in directory ${srcdir}" >&6;}
ac_aux_dir=
......@@ -12628,7 +12628,7 @@ fi
 
# Define the identity of the package.
PACKAGE='libmaus2'
VERSION='2.0.652'
VERSION='2.0.656'
 
 
cat >>confdefs.h <<_ACEOF
......@@ -28157,7 +28157,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by libmaus2 $as_me 2.0.652, which was
This file was extended by libmaus2 $as_me 2.0.656, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
CONFIG_FILES = $CONFIG_FILES
......@@ -28224,7 +28224,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
libmaus2 config.status 2.0.652
libmaus2 config.status 2.0.656
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
 
......
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.69 for libmaus2 2.0.653.
@%:@ Generated by GNU Autoconf 2.69 for libmaus2 2.0.657.
@%:@
@%:@ Report bugs to <tischler@mpi-cbg.de>.
@%:@
......@@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='libmaus2'
PACKAGE_TARNAME='libmaus2'
PACKAGE_VERSION='2.0.653'
PACKAGE_STRING='libmaus2 2.0.653'
PACKAGE_VERSION='2.0.657'
PACKAGE_STRING='libmaus2 2.0.657'
PACKAGE_BUGREPORT='tischler@mpi-cbg.de'
PACKAGE_URL='https://github.com/gt1/libmaus2'
 
......@@ -1609,7 +1609,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures libmaus2 2.0.653 to adapt to many kinds of systems.
\`configure' configures libmaus2 2.0.657 to adapt to many kinds of systems.
 
Usage: $0 [OPTION]... [VAR=VALUE]...
 
......@@ -1681,7 +1681,7 @@ fi
 
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of libmaus2 2.0.653:";;
short | recursive ) echo "Configuration of libmaus2 2.0.657:";;
esac
cat <<\_ACEOF
 
......@@ -1851,7 +1851,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
libmaus2 configure 2.0.653
libmaus2 configure 2.0.657
generated by GNU Autoconf 2.69
 
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -2724,7 +2724,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
 
It was created by libmaus2 $as_me 2.0.653, which was
It was created by libmaus2 $as_me 2.0.657, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
$ $0 $@
......@@ -3072,7 +3072,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
LIBRARY_VERSION=2:653:0
LIBRARY_VERSION=2:657:0
{ $as_echo "$as_me:${as_lineno-$LINENO}: Configuring for source in directory ${srcdir}" >&5
$as_echo "$as_me: Configuring for source in directory ${srcdir}" >&6;}
ac_aux_dir=
......@@ -12628,7 +12628,7 @@ fi
 
# Define the identity of the package.
PACKAGE='libmaus2'
VERSION='2.0.653'
VERSION='2.0.657'
 
 
cat >>confdefs.h <<_ACEOF
......@@ -28157,7 +28157,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by libmaus2 $as_me 2.0.653, which was
This file was extended by libmaus2 $as_me 2.0.657, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
CONFIG_FILES = $CONFIG_FILES
......@@ -28224,7 +28224,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
libmaus2 config.status 2.0.653
libmaus2 config.status 2.0.657
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
 
......
@%:@! /bin/sh
@%:@ Guess values for system-dependent variables and create Makefiles.
@%:@ Generated by GNU Autoconf 2.69 for libmaus2 2.0.653.
@%:@ Generated by GNU Autoconf 2.69 for libmaus2 2.0.657.
@%:@
@%:@ Report bugs to <tischler@mpi-cbg.de>.
@%:@
......@@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='libmaus2'
PACKAGE_TARNAME='libmaus2'
PACKAGE_VERSION='2.0.653'
PACKAGE_STRING='libmaus2 2.0.653'
PACKAGE_VERSION='2.0.657'
PACKAGE_STRING='libmaus2 2.0.657'
PACKAGE_BUGREPORT='tischler@mpi-cbg.de'
PACKAGE_URL='https://github.com/gt1/libmaus2'
 
......@@ -1609,7 +1609,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures libmaus2 2.0.653 to adapt to many kinds of systems.
\`configure' configures libmaus2 2.0.657 to adapt to many kinds of systems.
 
Usage: $0 [OPTION]... [VAR=VALUE]...
 
......@@ -1681,7 +1681,7 @@ fi
 
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of libmaus2 2.0.653:";;
short | recursive ) echo "Configuration of libmaus2 2.0.657:";;
esac
cat <<\_ACEOF
 
......@@ -1851,7 +1851,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
libmaus2 configure 2.0.653
libmaus2 configure 2.0.657
generated by GNU Autoconf 2.69
 
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -2724,7 +2724,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
 
It was created by libmaus2 $as_me 2.0.653, which was
It was created by libmaus2 $as_me 2.0.657, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
$ $0 $@
......@@ -3072,7 +3072,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
LIBRARY_VERSION=2:653:0
LIBRARY_VERSION=2:657:0
{ $as_echo "$as_me:${as_lineno-$LINENO}: Configuring for source in directory ${srcdir}" >&5
$as_echo "$as_me: Configuring for source in directory ${srcdir}" >&6;}
ac_aux_dir=
......@@ -12628,7 +12628,7 @@ fi
 
# Define the identity of the package.
PACKAGE='libmaus2'
VERSION='2.0.653'
VERSION='2.0.657'
 
 
cat >>confdefs.h <<_ACEOF
......@@ -28157,7 +28157,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by libmaus2 $as_me 2.0.653, which was
This file was extended by libmaus2 $as_me 2.0.657, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
CONFIG_FILES = $CONFIG_FILES
......@@ -28224,7 +28224,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
libmaus2 config.status 2.0.653
libmaus2 config.status 2.0.657
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
 
......
This diff is collapsed.
......@@ -4,7 +4,7 @@ m4trace:aclocal.m4:1460: -1- m4_include([m4/ltoptions.m4])
m4trace:aclocal.m4:1461: -1- m4_include([m4/ltsugar.m4])
m4trace:aclocal.m4:1462: -1- m4_include([m4/ltversion.m4])
m4trace:aclocal.m4:1463: -1- m4_include([m4/lt~obsolete.m4])
m4trace:configure.ac:1: -1- AC_INIT([libmaus2], [2.0.653], [tischler@mpi-cbg.de], [libmaus2], [https://github.com/gt1/libmaus2])
m4trace:configure.ac:1: -1- AC_INIT([libmaus2], [2.0.657], [tischler@mpi-cbg.de], [libmaus2], [https://github.com/gt1/libmaus2])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^_?A[CHUM]_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([_AC_])
m4trace:configure.ac:1: -1- m4_pattern_forbid([^LIBOBJS$], [do not use LIBOBJS directly, use AC_LIBOBJ (see section `AC_LIBOBJ vs LIBOBJS'])
......
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for libmaus2 2.0.653.
# Generated by GNU Autoconf 2.69 for libmaus2 2.0.657.
#
# Report bugs to <tischler@mpi-cbg.de>.
#
......@@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='libmaus2'
PACKAGE_TARNAME='libmaus2'
PACKAGE_VERSION='2.0.653'
PACKAGE_STRING='libmaus2 2.0.653'
PACKAGE_VERSION='2.0.657'
PACKAGE_STRING='libmaus2 2.0.657'
PACKAGE_BUGREPORT='tischler@mpi-cbg.de'
PACKAGE_URL='https://github.com/gt1/libmaus2'
 
......@@ -1609,7 +1609,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures libmaus2 2.0.653 to adapt to many kinds of systems.
\`configure' configures libmaus2 2.0.657 to adapt to many kinds of systems.
 
Usage: $0 [OPTION]... [VAR=VALUE]...
 
......@@ -1681,7 +1681,7 @@ fi
 
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of libmaus2 2.0.653:";;
short | recursive ) echo "Configuration of libmaus2 2.0.657:";;
esac
cat <<\_ACEOF
 
......@@ -1851,7 +1851,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
libmaus2 configure 2.0.653
libmaus2 configure 2.0.657
generated by GNU Autoconf 2.69
 
Copyright (C) 2012 Free Software Foundation, Inc.
......@@ -2724,7 +2724,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
 
It was created by libmaus2 $as_me 2.0.653, which was
It was created by libmaus2 $as_me 2.0.657, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
$ $0 $@
......@@ -3072,7 +3072,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
LIBRARY_VERSION=2:653:0
LIBRARY_VERSION=2:657:0
{ $as_echo "$as_me:${as_lineno-$LINENO}: Configuring for source in directory ${srcdir}" >&5
$as_echo "$as_me: Configuring for source in directory ${srcdir}" >&6;}
ac_aux_dir=
......@@ -12628,7 +12628,7 @@ fi
 
# Define the identity of the package.
PACKAGE='libmaus2'
VERSION='2.0.653'
VERSION='2.0.657'
 
 
cat >>confdefs.h <<_ACEOF
......@@ -28157,7 +28157,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by libmaus2 $as_me 2.0.653, which was
This file was extended by libmaus2 $as_me 2.0.657, which was
generated by GNU Autoconf 2.69. Invocation command line was
 
CONFIG_FILES = $CONFIG_FILES
......@@ -28224,7 +28224,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
libmaus2 config.status 2.0.653
libmaus2 config.status 2.0.657
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
 
......
AC_INIT(libmaus2,2.0.653,[tischler@mpi-cbg.de],[libmaus2],[https://github.com/gt1/libmaus2])
LIBRARY_VERSION=2:653:0
AC_INIT(libmaus2,2.0.657,[tischler@mpi-cbg.de],[libmaus2],[https://github.com/gt1/libmaus2])
LIBRARY_VERSION=2:657:0
AC_MSG_NOTICE([Configuring for source in directory ${srcdir}])
AC_CANONICAL_SYSTEM
AC_CANONICAL_HOST
......
libmaus2 (2.0.657-1) unstable; urgency=medium
* New upstream version
-- Andreas Tille <tille@debian.org> Mon, 19 Aug 2019 18:18:32 +0200
libmaus2 (2.0.653-1) unstable; urgency=medium
* New upstream version
......
......@@ -1773,7 +1773,9 @@ libmaus2rmq_include_HEADERS=\
libmaus2network_includedir=$(includedir)/libmaus2/network
libmaus2network_include_HEADERS=\
libmaus2/network/Socket.hpp \
libmaus2/network/SocketBase.hpp \
libmaus2/network/ClientSocket.hpp \
libmaus2/network/ServerSocket.hpp \
libmaus2/network/GetHostName.hpp \
libmaus2/network/SocketInputStreamBuffer.hpp \
libmaus2/network/SocketInputStream.hpp \
......
......@@ -4173,7 +4173,9 @@ libmaus2rmq_include_HEADERS = \
 
libmaus2network_includedir = $(includedir)/libmaus2/network
libmaus2network_include_HEADERS = \
libmaus2/network/Socket.hpp \
libmaus2/network/SocketBase.hpp \
libmaus2/network/ClientSocket.hpp \
libmaus2/network/ServerSocket.hpp \
libmaus2/network/GetHostName.hpp \
libmaus2/network/SocketInputStreamBuffer.hpp \
libmaus2/network/SocketInputStream.hpp \
......@@ -22,6 +22,7 @@
#include <libmaus2/autoarray/AutoArray.hpp>
#include <libmaus2/util/TempFileRemovalContainer.hpp>
#include <libmaus2/sorting/SortingBufferedOutputFile.hpp>
#include <libmaus2/util/Concat.hpp>
namespace libmaus2
{
......
This diff is collapsed.
......@@ -33,12 +33,8 @@ namespace libmaus2
{
namespace bambam
{
struct ReadEndsBlockIndexSet
struct ReadEndsBlockIndexSetBase
{
typedef ReadEndsBlockIndexSet this_type;
typedef libmaus2::util::unique_ptr<this_type>::type unique_ptr_type;
typedef libmaus2::util::shared_ptr<this_type>::type shared_ptr_type;
typedef libmaus2::index::ExternalMemoryIndexDecoder<
libmaus2::bambam::ReadEndsBase,
libmaus2::bambam::ReadEndsContainerBase::baseIndexShift,
......@@ -54,55 +50,124 @@ namespace libmaus2
return numblocks;
}
std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const info;
uint64_t const numblocks;
std::vector<uint64_t> O;
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> indexstreams;
libmaus2::autoarray::AutoArray<index_decoder_pointer_type> indexes;
ReadEndsBlockIndexSet(std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const & rinfo)
: info(rinfo), numblocks(computeNumBlocks(rinfo)), O(rinfo.size()), indexstreams(rinfo.size()), indexes(numblocks)
static std::vector<uint64_t> computeO(std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const & info)
{
std::vector<uint64_t> O(info.size());
for ( uint64_t i = 1; i < O.size(); ++i )
O[i] = O[i-1] + rinfo[i-1].indexoffset.size();
O[i] = O[i-1] + info[i-1].indexoffset.size();
for ( uint64_t i = 0; i < rinfo.size(); ++i )
return O;
}
static libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type>::unique_ptr_type openIndexStreams(
std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const & info
)
{
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type>::unique_ptr_type indexstreams(
new libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type>(info.size())
);
for ( uint64_t i = 0; i < info.size(); ++i )
{
libmaus2::aio::InputStreamInstance::unique_ptr_type tptr(
new libmaus2::aio::InputStreamInstance(rinfo[i].indexfilename)
new libmaus2::aio::InputStreamInstance(info[i].indexfilename)
);
(*indexstreams)[i] = UNIQUE_PTR_MOVE(tptr);
}
return indexstreams;
}
static uint64_t getBaseBlockSize()
{
return libmaus2::bambam::ReadEndsContainerBase::baseIndexStep;
}
static libmaus2::autoarray::AutoArray<index_decoder_pointer_type>::unique_ptr_type openIndexes(
std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const & info,
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type>::unique_ptr_type & indexstreams,
uint64_t const numblocks,
std::vector<uint64_t> const & O
)
{
libmaus2::autoarray::AutoArray<index_decoder_pointer_type>::unique_ptr_type indexes(
new libmaus2::autoarray::AutoArray<index_decoder_pointer_type>(numblocks)
);
indexstreams[i] = UNIQUE_PTR_MOVE(tptr);
for ( uint64_t j = 0; j < rinfo[i].indexoffset.size(); ++j )
for ( uint64_t i = 0; i < indexstreams->size(); ++i )
{
std::istream & str = *((*indexstreams)[i]);
for ( uint64_t j = 0; j < info[i].indexoffset.size(); ++j )
{
indexstreams[i]->clear();
indexstreams[i]->seekg(rinfo[i].indexoffset[j]);
str.clear();
str.seekg(info[i].indexoffset[j]);
index_decoder_pointer_type iptr(
new index_decoder_type(*(indexstreams[i]))
new index_decoder_type(str)
);
indexes[O[i] + j] = UNIQUE_PTR_MOVE(iptr);
(*indexes)[O[i] + j] = UNIQUE_PTR_MOVE(iptr);
}
}
return indexes;
}
};
uint64_t size() const
struct ReadEndsBlockIndexSetData : public ReadEndsBlockIndexSetBase
{
return numblocks;
}
typedef ReadEndsBlockIndexSetData this_type;
typedef libmaus2::util::unique_ptr<this_type>::type unique_ptr_type;
typedef libmaus2::util::shared_ptr<this_type>::type shared_ptr_type;
private:
index_decoder_type & operator[](uint64_t i)
{
return *(indexes[i]);
return *((*indexes)[i]);
}
public:
std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const info;
uint64_t const numblocks;
std::vector<uint64_t> const O;
private:
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type>::unique_ptr_type indexstreams;
libmaus2::autoarray::AutoArray<index_decoder_pointer_type>::unique_ptr_type indexes;
libmaus2::parallel::StdMutex mutex;
public:
ReadEndsBlockIndexSetData(std::vector<ReadEndsBlockDecoderBaseCollectionInfoBase> const & rinfo)
: info(rinfo), numblocks(computeNumBlocks(rinfo)), O(computeO(rinfo)), indexstreams(openIndexStreams(rinfo)), indexes(openIndexes(rinfo,indexstreams,numblocks,O)), mutex()
{
}
uint64_t size() const
{
return numblocks;
}
std::pair<uint64_t,uint64_t> getOffset(uint64_t const block, uint64_t const subblock)
{
return (*this)[block][subblock];
std::pair<uint64_t,uint64_t> P;
{
libmaus2::parallel::ScopeStdMutex smutex(mutex);
P = (*this)[block][subblock];
}
return P;
}
};
static uint64_t getBaseBlockSize()
struct ReadEndsBlockIndexSet : public ReadEndsBlockIndexSetBase
{
typedef ReadEndsBlockIndexSet this_type;
typedef libmaus2::util::unique_ptr<this_type>::type unique_ptr_type;
typedef libmaus2::util::shared_ptr<this_type>::type shared_ptr_type;
ReadEndsBlockIndexSetData & data;
ReadEndsBlockIndexSet(ReadEndsBlockIndexSetData & rdata) : data(rdata)
{
return libmaus2::bambam::ReadEndsContainerBase::baseIndexStep;
}
std::pair<uint64_t,uint64_t> merge(
......@@ -119,29 +184,29 @@ namespace libmaus2
exp += V[i].second-V[i].first;
}
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> datastreams(info.size());
libmaus2::autoarray::AutoArray<libmaus2::lz::SnappyInputStream::unique_ptr_type> zdatastreams(size());
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> datastreams(data.info.size());
libmaus2::autoarray::AutoArray<libmaus2::lz::SnappyInputStream::unique_ptr_type> zdatastreams(data.size());
libmaus2::bambam::ReadEnds R;
for ( uint64_t i = 0; i < info.size(); ++i )
for ( uint64_t i = 0; i < data.info.size(); ++i )
{
// open data stream
libmaus2::aio::InputStreamInstance::unique_ptr_type tptr(
new libmaus2::aio::InputStreamInstance(info[i].datafilename)
new libmaus2::aio::InputStreamInstance(data.info[i].datafilename)
);
datastreams[i] = UNIQUE_PTR_MOVE(tptr);
// set up uncompressors
for ( uint64_t j = 0; j < info[i].indexoffset.size(); ++j )
for ( uint64_t j = 0; j < data.info[i].indexoffset.size(); ++j )
{
uint64_t const blockid = O[i] + j;
uint64_t const blockid = data.O[i] + j;
uint64_t const vlow = V[blockid].first;
uint64_t const vhigh = V[blockid].second;
uint64_t const subblockid = vlow / getBaseBlockSize();
if ( vlow < vhigh )
{
std::pair<uint64_t,uint64_t> const zoffset = getOffset(blockid,subblockid);
std::pair<uint64_t,uint64_t> const zoffset = data.getOffset(blockid,subblockid);
libmaus2::lz::SnappyInputStream::unique_ptr_type zptr(
new libmaus2::lz::SnappyInputStream(
......@@ -234,29 +299,29 @@ namespace libmaus2
exp += V[i].second-V[i].first;
}
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> datastreams(info.size());
libmaus2::autoarray::AutoArray<libmaus2::lz::SnappyInputStream::unique_ptr_type> zdatastreams(size());
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> datastreams(data.info.size());
libmaus2::autoarray::AutoArray<libmaus2::lz::SnappyInputStream::unique_ptr_type> zdatastreams(data.size());
libmaus2::bambam::ReadEnds R;
for ( uint64_t i = 0; i < info.size(); ++i )
for ( uint64_t i = 0; i < data.info.size(); ++i )
{
// open data stream
libmaus2::aio::InputStreamInstance::unique_ptr_type tptr(
new libmaus2::aio::InputStreamInstance(info[i].datafilename)
new libmaus2::aio::InputStreamInstance(data.info[i].datafilename)
);
datastreams[i] = UNIQUE_PTR_MOVE(tptr);
// set up uncompressors
for ( uint64_t j = 0; j < info[i].indexoffset.size(); ++j )
for ( uint64_t j = 0; j < data.info[i].indexoffset.size(); ++j )
{
uint64_t const blockid = O[i] + j;
uint64_t const blockid = data.O[i] + j;
uint64_t const vlow = V[blockid].first;
uint64_t const vhigh = V[blockid].second;
if ( vlow < vhigh )
{
uint64_t const subblockid = vlow / getBaseBlockSize();
std::pair<uint64_t,uint64_t> const zoffset = getOffset(blockid,subblockid);
std::pair<uint64_t,uint64_t> const zoffset = data.getOffset(blockid,subblockid);
libmaus2::lz::SnappyInputStream::unique_ptr_type zptr(
new libmaus2::lz::SnappyInputStream(
......@@ -367,29 +432,29 @@ namespace libmaus2
exp += V[i].second-V[i].first;
}
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> datastreams(info.size());
libmaus2::autoarray::AutoArray<libmaus2::lz::SnappyInputStream::unique_ptr_type> zdatastreams(size());
libmaus2::autoarray::AutoArray<libmaus2::aio::InputStreamInstance::unique_ptr_type> datastreams(data.info.size());
libmaus2::autoarray::AutoArray<libmaus2::lz::SnappyInputStream::unique_ptr_type> zdatastreams(data.size());
libmaus2::bambam::ReadEnds R;
for ( uint64_t i = 0; i < info.size(); ++i )
for ( uint64_t i = 0; i < data.info.size(); ++i )
{
// open data stream
libmaus2::aio::InputStreamInstance::unique_ptr_type tptr(
new libmaus2::aio::InputStreamInstance(info[i].datafilename)
new libmaus2::aio::InputStreamInstance(data.info[i].datafilename)
);
datastreams[i] = UNIQUE_PTR_MOVE(tptr);
// set up uncompressors
for ( uint64_t j = 0; j < info[i].indexoffset.size(); ++j )
for ( uint64_t j = 0; j < data.info[i].indexoffset.size(); ++j )
{
uint64_t const blockid = O[i] + j;
uint64_t const blockid = data.O[i] + j;
uint64_t const vlow = V[blockid].first;
uint64_t const vhigh = V[blockid].second;
if ( vlow < vhigh )
{
uint64_t const subblockid = vlow / getBaseBlockSize();
std::pair<uint64_t,uint64_t> const zoffset = getOffset(blockid,subblockid);
std::pair<uint64_t,uint64_t> const zoffset = data.getOffset(blockid,subblockid);
libmaus2::lz::SnappyInputStream::unique_ptr_type zptr(
new libmaus2::lz::SnappyInputStream(
......
......@@ -420,6 +420,8 @@ namespace libmaus2
int verbose;
uint64_t const fragmergepar;
void freeBuffers()
{
fragmentBufferFreeListPreSort.reset();
......@@ -466,6 +468,7 @@ namespace libmaus2
libmaus2::parallel::SimpleThreadPool & rSTP,
std::istream & in,
int const level,
uint64_t const rfragmergepar,
std::string const & rtempfileprefix,
std::string const & rhash,
bool const rfixmates = true,
......@@ -546,7 +549,8 @@ namespace libmaus2
unmergeFragReadEndsRegions(0),
unmergePairReadEndsRegions(0),
hash(rhash),
verbose(0)
verbose(0),
fragmergepar(rfragmergepar)
{
STP.registerDispatcher(GICRPDid,&GICRPD);
STP.registerDispatcher(GIBDWPDid,&GIBDWPD);
......@@ -1284,7 +1288,9 @@ namespace libmaus2
}
}
void enqueMergeFragReadEndsLists()
void enqueMergeFragReadEndsLists(
libmaus2::parallel::StdSemaphore::shared_ptr_type psem
)
{
if ( verbose )
{
......@@ -1329,11 +1335,14 @@ namespace libmaus2
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "\t\t(enque FragReadEndsMergeWorkPackage)\n";
}
libmaus2::bambam::ReadEndsBlockIndexSetData::shared_ptr_type setData(
new libmaus2::bambam::ReadEndsBlockIndexSetData(*MI)
);
for ( uint64_t i = 0; i < SMI.size(); ++i )
{
ReadEndsMergeRequest req(Pdupbitvec.get(),MI,SMI[i]);
FragReadEndsMergeWorkPackage * package = fragReadEndsMergeWorkPackages.getPackage();
*package = FragReadEndsMergeWorkPackage(req,0/*prio*/,FREMWPDid);
*package = FragReadEndsMergeWorkPackage(req,setData,psem,0/*prio*/,FREMWPDid);
package->subid = i;
STP.enque(package);
}
......@@ -1344,7 +1353,9 @@ namespace libmaus2
}
}
void enqueMergePairReadEndsLists()
void enqueMergePairReadEndsLists(
libmaus2::parallel::StdSemaphore::shared_ptr_type psem
)
{
if ( verbose )
{
......@@ -1389,11 +1400,14 @@ namespace libmaus2
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "\t\t(enque PairReadEndsMergeWorkPackage)\n";
}
libmaus2::bambam::ReadEndsBlockIndexSetData::shared_ptr_type setData(
new libmaus2::bambam::ReadEndsBlockIndexSetData(*MI)
);
for ( uint64_t i = 0; i < SMI.size(); ++i )
{
ReadEndsMergeRequest req(Pdupbitvec.get(),MI,SMI[i]);
PairReadEndsMergeWorkPackage * package = pairReadEndsMergeWorkPackages.getPackage();
*package = PairReadEndsMergeWorkPackage(req,0/*prio*/,PREMWPDid);
*package = PairReadEndsMergeWorkPackage(req,setData,psem,0/*prio*/,PREMWPDid);
package->subid = i;
STP.enque(package);
}
......@@ -1784,6 +1798,13 @@ namespace libmaus2
void mergeReadEndsLists(std::ostream & metricsstr, std::string const progname)
{
// set up semaphore to limit parallelism
libmaus2::parallel::StdSemaphore::shared_ptr_type psem(
new libmaus2::parallel::StdSemaphore
);
for ( uint64_t i = 0; i < fragmergepar; ++i )
psem->post();
// set up duplicate data structure
uint64_t const ureadsParsed = static_cast<uint64_t>(readsParsed);
libmaus2::bitio::BitVector::unique_ptr_type Tdupbitvec(new libmaus2::bitio::BitVector(ureadsParsed));
......@@ -1797,7 +1818,7 @@ namespace libmaus2
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "\n\t(call enqueMergeFragReadEndsLists)\n";
}
enqueMergeFragReadEndsLists();
enqueMergeFragReadEndsLists(psem);
if ( verbose )
{
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
......@@ -1809,7 +1830,7 @@ namespace libmaus2
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "\t(call enqueMergePairReadEndsLists)\n";
}
enqueMergePairReadEndsLists();
enqueMergePairReadEndsLists(psem);
if ( verbose )
{
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
......
......@@ -21,6 +21,7 @@
#include <libmaus2/parallel/SimpleThreadWorkPackage.hpp>
#include <libmaus2/bambam/parallel/ReadEndsMergeRequest.hpp>
#include <libmaus2/bambam/ReadEndsBlockIndexSet.hpp>
namespace libmaus2
{
......@@ -35,6 +36,8 @@ namespace libmaus2
typedef libmaus2::util::shared_ptr<this_type>::type shared_ptr_type;
ReadEndsMergeRequest REQ;
libmaus2::bambam::ReadEndsBlockIndexSetData::shared_ptr_type setData;
libmaus2::parallel::StdSemaphore::shared_ptr_type psem;
FragReadEndsMergeWorkPackage() : libmaus2::parallel::SimpleThreadWorkPackage(), REQ()
{
......@@ -42,11 +45,13 @@ namespace libmaus2
}
FragReadEndsMergeWorkPackage(
ReadEndsMergeRequest const & rREQ,
libmaus2::bambam::ReadEndsBlockIndexSetData::shared_ptr_type rsetData,
libmaus2::parallel::StdSemaphore::shared_ptr_type rpsem,
uint64_t const rpriority,
uint64_t const rdispatcherid,
uint64_t const rpackageid = 0
)
: libmaus2::parallel::SimpleThreadWorkPackage(rpriority,rdispatcherid,rpackageid), REQ(rREQ)
: libmaus2::parallel::SimpleThreadWorkPackage(rpriority,rdispatcherid,rpackageid), REQ(rREQ), setData(rsetData), psem(rpsem)
{
}
......@@ -58,6 +63,8 @@ namespace libmaus2
{
libmaus2::parallel::SimpleThreadWorkPackage::operator=(O);
REQ = O.REQ;
setData = O.setData;
psem = O.psem;
}
return *this;
}
......
......@@ -61,17 +61,23 @@ namespace libmaus2
virtual void dispatch(libmaus2::parallel::SimpleThreadWorkPackage * P, libmaus2::parallel::SimpleThreadPoolInterfaceEnqueTermInterface & /* tpi */)
{
FragReadEndsMergeWorkPackage * BP = NULL;
bool gotsem = false;
try
{
FragReadEndsMergeWorkPackage * BP = dynamic_cast<FragReadEndsMergeWorkPackage *>(P);
BP = dynamic_cast<FragReadEndsMergeWorkPackage *>(P);
assert ( BP );
BP->psem->wait();
gotsem = true;
if ( verbose )
{
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "FragReadEndsMergeWorkPackageDispatcher package " << P << " instantiating ReadEndsBlockIndexSet" << std::endl;
}
ReadEndsBlockIndexSet fragindexset(*(BP->REQ.MI));
ReadEndsBlockIndexSet fragindexset(*(BP->setData));
if ( verbose )
{
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
......@@ -119,6 +125,10 @@ namespace libmaus2
}
mergeFinishedInterface.fragReadEndsMergeWorkPackageFinished(BP);
BP->psem->post();
gotsem = false;
packageReturnInterface.fragReadEndsMergeWorkPackageReturn(BP);
if ( verbose )
......@@ -129,6 +139,9 @@ namespace libmaus2
}
catch(std::exception const & ex)
{
if ( BP && gotsem )
BP->psem->post();
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "FragReadEndsMergeWorkPackageDispatcher package " << P << " exception: " << ex.what() << std::endl;
throw;
......
......@@ -21,6 +21,7 @@
#include <libmaus2/bambam/parallel/ReadEndsMergeRequest.hpp>
#include <libmaus2/parallel/SimpleThreadWorkPackage.hpp>
#include <libmaus2/bambam/ReadEndsBlockIndexSet.hpp>
namespace libmaus2
{
......@@ -35,18 +36,22 @@ namespace libmaus2
typedef libmaus2::util::shared_ptr<this_type>::type shared_ptr_type;
ReadEndsMergeRequest REQ;
libmaus2::bambam::ReadEndsBlockIndexSetData::shared_ptr_type setData;
libmaus2::parallel::StdSemaphore::shared_ptr_type psem;
PairReadEndsMergeWorkPackage() : libmaus2::parallel::SimpleThreadWorkPackage(), REQ()
PairReadEndsMergeWorkPackage() : libmaus2::parallel::SimpleThreadWorkPackage(), REQ(), setData()
{
}
PairReadEndsMergeWorkPackage(
ReadEndsMergeRequest const & rREQ,
libmaus2::bambam::ReadEndsBlockIndexSetData::shared_ptr_type rsetData,
libmaus2::parallel::StdSemaphore::shared_ptr_type rpsem,
uint64_t const rpriority,
uint64_t const rdispatcherid,
uint64_t const rpackageid = 0
)
: libmaus2::parallel::SimpleThreadWorkPackage(rpriority,rdispatcherid,rpackageid), REQ(rREQ)
: libmaus2::parallel::SimpleThreadWorkPackage(rpriority,rdispatcherid,rpackageid), REQ(rREQ), setData(rsetData), psem(rpsem)
{
}
......@@ -58,6 +63,8 @@ namespace libmaus2
{
libmaus2::parallel::SimpleThreadWorkPackage::operator=(O);
REQ = O.REQ;
setData = O.setData;
psem = O.psem;
}
return *this;
}
......
......@@ -64,17 +64,23 @@ namespace libmaus2
virtual void dispatch(libmaus2::parallel::SimpleThreadWorkPackage * P, libmaus2::parallel::SimpleThreadPoolInterfaceEnqueTermInterface & /* tpi */)
{
PairReadEndsMergeWorkPackage * BP = NULL;
bool gotsem = false;
try
{
PairReadEndsMergeWorkPackage * BP = dynamic_cast<PairReadEndsMergeWorkPackage *>(P);
BP = dynamic_cast<PairReadEndsMergeWorkPackage *>(P);
assert ( BP );
BP->psem->wait();
gotsem = true;
if ( verbose )
{
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "PairReadEndsMergeWorkPackageDispatcher package " << P << " instantiating ReadEndsBlockIndexSet" << std::endl;
}
ReadEndsBlockIndexSet pairindexset(*(BP->REQ.MI));
ReadEndsBlockIndexSet pairindexset(*(BP->setData));
if ( verbose )
{
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
......@@ -123,6 +129,10 @@ namespace libmaus2
}
mergeFinishedInterface.pairReadEndsMergeWorkPackageFinished(BP);
BP->psem->post();
gotsem = false;
packageReturnInterface.pairReadEndsMergeWorkPackageReturn(BP);
if ( verbose )
......@@ -133,6 +143,9 @@ namespace libmaus2
}
catch(std::exception const & ex)
{
if ( BP && gotsem )
BP->psem->post();
libmaus2::parallel::ScopeStdSpinLock slock(libmaus2::aio::StreamLock::cerrlock);
std::cerr << "PairReadEndsMergeWorkPackageDispatcher exception: " << ex.what() << std::endl;
throw;
......
/*
libmaus2
Copyright (C) 2009-2019 German Tischler
Copyright (C) 2011-2013 Genome Research Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#if ! defined(LIBMAUS2_CLIENTSOCKET_HPP)
#define LIBMAUS2_CLIENTSOCKET_HPP
#include <libmaus2/network/SocketBase.hpp>
namespace libmaus2
{
namespace network
{
struct ClientSocket : public SocketBase
{
typedef ClientSocket this_type;
typedef ::libmaus2::util::unique_ptr<this_type>::type unique_ptr_type;
ClientSocket(unsigned short port, char const * hostname)
: SocketBase()
{
setAddress(hostname, remaddr);
remaddr.sin_port = htons(port);
remaddrset = true;
while ( ::connect(getFD(),reinterpret_cast<const sockaddr *>(&remaddr),sizeof(remaddr)) != 0 )
{
int const error = errno;
switch ( error )
{
case EINTR:
case EAGAIN:
break;
default:
{
::libmaus2::exception::LibMausException se;
se.getStream() << "ClientSocket: connect() failed with error " << strerror(error) << std::endl;
se.finish();
throw se;
}
}
}
}
static SocketBase::unique_ptr_type baseCast(ClientSocket::unique_ptr_type & C)
{
int const fd = C->getFD();
C->releaseFD();
try
{
SocketBase::unique_ptr_type ptr ( new SocketBase(fd) );
return ptr;
}
catch(...)
{
::close(fd);
throw;
}
}
static SocketBase::unique_ptr_type baseAlloc(unsigned short const port, char const * const hostname)
{
ClientSocket::unique_ptr_type ptr ( new ClientSocket(port,hostname) );
SocketBase::unique_ptr_type baseptr = baseCast(ptr);
return baseptr;
}
static SocketBase::unique_ptr_type baseAlloc(unsigned short const port, std::string const & hostname)
{
SocketBase::unique_ptr_type baseptr = baseAlloc(port,hostname.c_str());
return baseptr;
}
};
}
}
#endif