Commit d6238780 authored by Steffen Möller's avatar Steffen Möller
Browse files

New upstream version 1.4.1+git20191130.664696c+dfsg

parent 8febe06b

gatb-core/CMakeLists.txt

+0 −2
Original line number Diff line number Diff line
@@ -110,8 +110,6 @@ if (debug) 
    set (LIBRARY_COMPILE_DEFINITIONS  "${LIBRARY_COMPILE_DEFINITIONS} -g -p ${LIB_COMPILE_WARNINGS}")

    set (CMAKE_BUILD_TYPE Debug) # else CMake adds DNDEBUG

    message("-- COMPILATION IN DEBUG MODE")

else()

    set (LIBRARY_COMPILE_DEFINITIONS  "${LIBRARY_COMPILE_DEFINITIONS} -O3 -DNDEBUG ${LIB_COMPILE_WARNINGS}")

endif()



if (INT128_FOUND)

gatb-core/doc/doxygen/src/dbgh5page.hpp

+2 −0
Original line number Diff line number Diff line
@@ -91,6 +91,8 @@ 
          -verbose           (1 arg) :    verbosity level  [default '1']

          -email             (1 arg) :    send statistics to the given email address  [default '']

          -email-fmt         (1 arg) :    'raw' or 'xml'  [default 'raw']

          -edge-km           (1 arg) :    Kececioglu-Myers edge representation  [default '0']

 

 * \endcode

 *

 *

gatb-core/src/gatb/bank/impl/BankFasta.cpp

+1 −0
Original line number Diff line number Diff line
@@ -651,6 +651,7 @@ void BankFasta::Iterator::init () 
        *bf = (buffered_file_t *)  CALLOC (1, sizeof(buffered_file_t));

        (*bf)->buffer = (unsigned char*)  MALLOC (BUFFER_SIZE);

        (*bf)->stream = gzopen (fname, "r");

        gzbuffer((*bf)->stream,2*1024*1024);

		

        /** We check that we can open the file. */

        if ((*bf)->stream == NULL)

gatb-core/src/gatb/bcalm2/bglue_algo.cpp

+35 −23
Original line number Diff line number Diff line
@@ -63,12 +63,9 @@ using namespace gatb::core::tools::collections::impl; 
using namespace std;



// let's be clear here:

// UF hashes will be stored in 32 bits for efficiency (as I don't want to have a 64-bits UF for memory reasons, also, would require to modify unionFind.hpp)

typedef uint32_t uf_hashes_t;

// but there can be more than 2^{32} sequences in the glue file

typedef uint64_t uf_hashes_t; // UF hashes are the hash values of k-mers to be inserted into the UF data structure. Don't try setting to uint32_t, would be a disaster

typedef uint64_t seq_idx_t;

// so, potentially, more than 2^{32} UF hashes (but not necessarily, consider that some sequences don't need to be glued)

// what will happen is that more one UF class won't be linked to a single unitig, but multiple unitigs

typedef uint32_t uf_class_t; // UF class is the identifier of an element in the UF

// let's hope that there won't be saturation (only 1 UF class with all unitigs)

// if this happens, then "Top 10 glue partitions by size:" will show only one entry and BCALM will blow up in memory

// a fix would be to use a 64 bits UF (to be coded later)
@@ -197,6 +194,24 @@ static string skip_first_abundance(const string& list) 
    return res;

}



static string make_header(const int seq_size, const string& abundances, bool all_abundance_counts)

{

    string header;

    float mean_abundance = get_mean_abundance(abundances);

    uint64_t sum_abundances = get_sum_abundance(abundances);

    if (all_abundance_counts)

    {

        // in this setting, all kmer wabundances are printed in the order of the kmers in the sequence

        header = "LN:i:" + to_string(seq_size) + " ab:Z:" + abundances;

    }

    else

    {

        // km is not a standard GFA field so i'm putting it in lower case as per the spec

        header = "LN:i:" + to_string(seq_size) + " KC:i:" + to_string(sum_abundances) + " km:f:" + to_string_with_precision(mean_abundance);

    }

    return header;

}



template<int SPAN>

struct markedSeq

{
@@ -699,6 +714,7 @@ void bglue(Storage *storage, 
        int kmerSize, 

        int nb_glue_partitions, 

        int nb_threads, 

        bool all_abundance_counts,

        bool verbose

        )

{
@@ -804,7 +820,7 @@ void bglue(Storage *storage, 
    }



    // create a UF data structure

    // this one stores nb_uf_keys * uint64_t (actually, atomic's). so it's bigger than uf_hashes

    // this one stores nb_uf_keys * uint64_t (actually, atomic's).

    unionFind ufkmers(nb_uf_keys);



#if 0
@@ -911,13 +927,13 @@ void bglue(Storage *storage, 
    if (only_uf) // for debugging

        return;



    /* now we're mirroring the UF to a vector of uint32_t's, it will take less space, and strictly same information

    /* now we're mirroring the UF to a vector of uint32_t's (uf_class_t), it will take less space, and strictly same information

     * this is to get rid of the rank (one uint32) per element in the current UF implementation. 

     * To do this, we're using the disk to save space of populating one vector from the other in memory. 

     * (saves having to allocate both vectors at the same time) */

    

    BagFile<uf_hashes_t> *ufkmers_bagf = new BagFile<uf_hashes_t>(prefix+".glue.uf");  LOCAL(ufkmers_bagf);

	BagCache<uf_hashes_t> *ufkmers_bag = new BagCache<uf_hashes_t>(  ufkmers_bagf, 10000 );   LOCAL(ufkmers_bag);

    BagFile<uf_class_t> *ufkmers_bagf = new BagFile<uf_class_t>(prefix+".glue.uf");  LOCAL(ufkmers_bagf);

	BagCache<uf_class_t> *ufkmers_bag = new BagCache<uf_class_t>(  ufkmers_bagf, 10000 );   LOCAL(ufkmers_bag);



    for (unsigned long i = 0; i < nb_uf_keys; i++)

        //ufkmers_vector[i] = ufkmers.find(i); // just in-memory without the disk
@@ -930,15 +946,15 @@ void bglue(Storage *storage, 


    ufkmers_bag->flush();



    std::vector<uf_hashes_t> ufkmers_vector(nb_uf_keys);

    IteratorFile<uf_hashes_t> ufkmers_file(prefix+".glue.uf");

    std::vector<uf_class_t> ufkmers_vector(nb_uf_keys);

    IteratorFile<uf_class_t> ufkmers_file(prefix+".glue.uf");

    unsigned long i = 0;

    for (ufkmers_file.first(); !ufkmers_file.isDone(); ufkmers_file.next())

            ufkmers_vector[i++] = ufkmers_file.item();



    System::file().remove (prefix+".glue.uf");

    

    logging("loaded 32-bit UF (" + to_string(nb_uf_keys*sizeof(uf_hashes_t)/1024/1024) + " MB)");

    logging("loaded 32-bit UF (" + to_string(nb_uf_keys*sizeof(uf_class_t)/1024/1024) + " MB)");

  

    // setup output file

    string output_prefix = prefix;
@@ -1000,7 +1016,7 @@ void bglue(Storage *storage, 


    // partition the glue into many files, à la dsk

    auto partitionGlue = [k, &modelCanon /* crashes if copied!*/, \

        &get_UFclass, &gluePartitions,

        &get_UFclass, &gluePartitions, all_abundance_counts,

        &out, &outLock, &nb_seqs_in_partition, nbGluePartitions]

            (const Sequence& sequence)

    {
@@ -1024,11 +1040,8 @@ void bglue(Storage *storage, 
        if (!found_class) // this one doesn't need to be glued

        {

            const string abundances = comment.substr(3);

            float mean_abundance = get_mean_abundance(abundances);

            uint64_t sum_abundances = get_sum_abundance(abundances);

            

            // km is not a standard GFA field so i'm putting it in lower case as per the spec

            output(seq, out, "LN:i:" + to_string(seq.size()) + " KC:i:" + to_string(sum_abundances) + " km:f:" + to_string_with_precision(mean_abundance)); 

            string header = make_header(seq.size(),abundances, all_abundance_counts);

            output(seq, out, header); 

            return;

        }
@@ -1082,7 +1095,7 @@ void bglue(Storage *storage, 
    for (int partition = 0; partition < nbGluePartitions; partition++)

    {

        auto glue_partition = [&modelCanon, &ufkmers, partition, &gluePartition_prefix, nbGluePartitions, &copy_nb_seqs_in_partition,

        &get_UFclass, &out, &outLock, kmerSize]( int thread_id)

        &get_UFclass, &out, &outLock, kmerSize, all_abundance_counts]( int thread_id)

        {

            int k = kmerSize;
@@ -1172,10 +1185,9 @@ void bglue(Storage *storage, 
                string seq, abs;

                glue_sequences(seqs_to_glue[i], seqs_to_glue_is_circular[i], sequences, abundances, kmerSize, seq, abs); // takes as input the indices of ordered sequences, whether that sequence is circular, and the markedSeq's themselves along with their abundances



                float mean_abundance = get_mean_abundance(abs);

                uint32_t sum_abundances = get_sum_abundance(abs);

                {

                    output(seq, out, "LN:i:" + to_string(seq.size()) + " KC:i:" + to_string(sum_abundances) + " km:f:" + to_string_with_precision(mean_abundance));

                    string header = make_header(seq.size(),abs, all_abundance_counts);

                    output(seq, out, header);

                }

            }
@@ -1198,7 +1210,7 @@ void bglue(Storage *storage, 


    logging("end");



    bool debug_keep_glue_files = true; // for debugging // TODO enable it if -redo-bglue param was provided (need some info from UnitigsConstructionAlgorithm). 

    bool debug_keep_glue_files = false; // for debugging // TODO warning: if debug_keep_glue_files is set to 'false,' then the debug option '-redo-bglue' cannot work because it needs those bglue files

    if (debug_keep_glue_files)

    {

        std::cout << "debug: not deleting glue files" << std::endl;

gatb-core/src/gatb/bcalm2/bglue_algo.hpp

+1 −0
Original line number Diff line number Diff line
@@ -150,6 +150,7 @@ void bglue(gatb::core::tools::storage::impl::Storage* storage, 
        int kmerSize, 

        int nb_glue_partitions, 

        int nb_threads, 

        bool all_abundance_counts,

        bool verbose

        );