Commit d3e6b2a1 authored by Andreas Tille's avatar Andreas Tille
Browse files

New upstream version 0.0+git20180707.42b5cfd

parent 4a3900a3

src/malt/AlignmentEngine.java

+4 −4
Original line number Diff line number Diff line
@@ -23,7 +23,7 @@ import jloda.util.Basic; 
import malt.align.AlignerOptions;

import malt.align.BandedAligner;

import malt.data.*;

import malt.genes.GeneTableAccess;

import malt.genes.GeneItemAccessor;

import malt.io.*;

import malt.util.FixedSizePriorityQueue;

import malt.util.Utilities;
@@ -59,7 +59,7 @@ public class AlignmentEngine { 
    private final FileWriterRanked unalignedReadsWriter;

    private final RMA6Writer rmaWriter;



    private final GeneTableAccess geneTableAccess;

    private final GeneItemAccessor geneTableAccess;



    // parameters

    private final double minRawScore;
@@ -100,7 +100,7 @@ public class AlignmentEngine { 
    AlignmentEngine(final int threadNumber, final MaltOptions maltOptions, AlignerOptions alignerOptions, final ReferencesDBAccess referencesDB,

                    final ReferencesHashTableAccess[] tables, final FastAReader fastAReader,

                    final FileWriterRanked matchesWriter, final RMA6Writer rmaWriter,

                    final FileWriterRanked alignedReadsWriter, final FileWriterRanked unalignedReadsWriter, final GeneTableAccess geneTableAccess) throws IOException {

                    final FileWriterRanked alignedReadsWriter, final FileWriterRanked unalignedReadsWriter, final GeneItemAccessor geneTableAccess) throws IOException {

        this.threadNumber = threadNumber;

        this.maltOptions = maltOptions;

        this.referencesDB = referencesDB;

src/malt/MaltBuild.java

+45 −39
Original line number Diff line number Diff line
@@ -21,13 +21,16 @@ package malt; 


import jloda.util.*;

import malt.data.*;

import malt.genes.GeneTableBuilder;

import malt.genes.GeneItem;

import malt.genes.GeneItemCreator;

import malt.mapping.Mapping;

import malt.util.Utilities;

import megan.classification.Classification;

import megan.classification.ClassificationManager;

import megan.classification.IdMapper;

import megan.classification.IdParser;

import megan.tools.AAdderBuild;

import megan.util.interval.Interval;



import java.io.File;

import java.io.IOException;
@@ -38,6 +41,8 @@ import java.util.*; 
 * Daniel Huson, 8.2014

 */

public class MaltBuild {

    final public static String INDEX_CREATOR = "MALT";



    /**

     * run the program

     *
@@ -110,24 +115,30 @@ public class MaltBuild { 
            proteinReduction = "";



        options.comment("Classification:");

        final boolean parseTaxonNames = true;

        final boolean parseTaxonNames = options.getOption("-tn", "parseTaxonNames", "Parse taxon names", true);

        final String gi2TaxaFile = options.getOption("-g2t", "gi2taxa", "GI-to-Taxonomy mapping file", "");

        final String acc2TaxaFile = options.getOption("-a2t", "acc2taxa", "Accession-to-Taxonomy mapping file", "");

        final String synonyms2TaxaFile = options.getOption("-s2t", "syn2taxa", "Synonyms-to-Taxonomy mapping file", "");



        final Map<String, String> cName2GIFileName = new HashMap<>();

        final Map<String, String> cName2AcessionFileName = new HashMap<>();

        final Map<String, String> cName2SynonymsFileName = new HashMap<>();

        final HashMap<String, String> class2GIFile = new HashMap<>();

        final HashMap<String, String> class2AccessionFile = new HashMap<>();

        final HashMap<String, String> class2SynonymsFile = new HashMap<>();



        final Set<String> classificationsToUse = new TreeSet<>();



        for (String cName : ClassificationManager.getAllSupportedClassifications()) {

            cName2GIFileName.put(cName, options.getOption("-g2" + cName.toLowerCase(), "gi2" + cName.toLowerCase(), "GI-to-" + cName + " mapping file (deprecated)", ""));

            cName2AcessionFileName.put(cName, options.getOption("-a2" + cName.toLowerCase(), "acc2" + cName.toLowerCase(), "Accession-to-" + cName + " mapping file", ""));

            cName2SynonymsFileName.put(cName, options.getOption("-s2" + cName.toLowerCase(), "syn2" + cName.toLowerCase(), "Synonyms-to-" + cName + " mapping file", ""));



            if (cName2GIFileName.get(cName).length() > 0 || cName2AcessionFileName.get(cName).length() > 0 || cName2SynonymsFileName.get(cName).length() > 0)

        for (String cName : ClassificationManager.getAllSupportedClassificationsExcludingNCBITaxonomy()) {

            class2GIFile.put(cName, options.getOption("-g2" + cName.toLowerCase(), "gi2" + cName.toLowerCase(), "GI-to-" + cName + " mapping file", ""));

            class2AccessionFile.put(cName, options.getOption("-a2" + cName.toLowerCase(), "acc2" + cName.toLowerCase(), "Accession-to-" + cName + " mapping file", ""));

            class2SynonymsFile.put(cName, options.getOption("-s2" + cName.toLowerCase(), "syn2" + cName.toLowerCase(), "Synonyms-to-" + cName + " mapping file", ""));

            final String tags = options.getOption("-t4" + cName.toLowerCase(), "tags4" + cName.toLowerCase(), "Tags for " + cName + " id parsing (must set to activate id parsing)", "").trim();

            if (tags.length() > 0)

                ProgramProperties.put(cName + "Tags", tags);

            ProgramProperties.put(cName + "ParseIds", tags.length() > 0);

            // final boolean useLCA = options.getOption("-l_" + cName.toLowerCase(), "lca" + cName.toLowerCase(), "Use LCA for assigning to '" + cName + "', alternative: best hit", ProgramProperties.get(cName + "UseLCA", cName.equals(Classification.Taxonomy)));

            // ProgramProperties.put(cName + "UseLCA", useLCA);



            if (class2GIFile.get(cName).length() > 0 || class2AccessionFile.get(cName).length() > 0 || class2AccessionFile.get(cName).length() > 0)

                classificationsToUse.add(cName);



            if (cName.equalsIgnoreCase(Classification.Taxonomy))

                options.getOption("-tn", "parseTaxonNames", "Parse taxon names", true);

        }



        final boolean functionalClassification = !options.getOption("-nf", "noFun", "Turn off functional classifications for provided mapping files (set this when using GFF files for DNA references)", false);
@@ -139,10 +150,11 @@ public class MaltBuild { 
        final boolean saveFirstWordOfReferenceHeaderOnly = options.getOption("-fwo", "firstWordOnly", "Save only first word of reference header", false);

        final int randomSeed = options.getOption("rns", "random", "Random number generator seed", 666);

        final float hashTableLoadFactor = options.getOption("hsf", "hashScaleFactor", "Hash table scale factor", 0.9f, 0.1f, 1.0f);

        //final boolean buildTableInMemory = options.getOption("btm", "buildTableInMemory", "Build the hash table in memory and then save (more memory, much faster)", true);

        final boolean buildTableInMemory = true; // don't make this an option because it is really slow...

        final boolean buildTableInMemory = options.getOption("btm", "buildTableInMemory", "Build the hash table in memory and then save (uses more memory, is much faster)", true);

        final boolean doBuildTables = !options.getOption("!xX", "xSkipTable", "Don't recompute index and tables, just compute profile support", false);



        final boolean lookInside = options.getOption("-ex", "extraStrict", "When given an input directory, look inside every GFF file to check that it is indeed in GFF3 format", false);



        options.done();

        Basic.setDebugMode(options.isVerbose());
@@ -164,20 +176,7 @@ public class MaltBuild { 


            }

        }

        if (gffFiles.size() == 1) {

            final File file = new File(gffFiles.get(0));

            if (file.isDirectory()) {

                System.err.println("Looking for GFF files in directory: " + file);

                gffFiles.clear();

                for (File aFile : Basic.getAllFilesInDirectory(file, new GFF3FileFilter(), true)) {

                    gffFiles.add(aFile.getPath());

                }

                if (gffFiles.size() == 0)

                    throw new IOException("No GFF files found in directory: " + file);

                else

                    System.err.println(String.format("Found: %,d", gffFiles.size()));

            }

        }

        AAdderBuild.setupGFFFiles(gffFiles, lookInside);



        System.err.println("Reference sequence type set to: " + sequenceType.toString());

        final IAlphabet referenceAlphabet;
@@ -240,7 +239,9 @@ public class MaltBuild { 
            }

        }



        // setup classification support

        if (gi2TaxaFile.length() > 0 || acc2TaxaFile.length() > 0 || synonyms2TaxaFile.length() > 0)

            classificationsToUse.add(Classification.Taxonomy);



        for (String cName : classificationsToUse) {

            final String cNameLowerCase = cName.toLowerCase();

            final String sourceName = (cName.equals(Classification.Taxonomy) ? "ncbi" : cNameLowerCase);
@@ -250,12 +251,12 @@ public class MaltBuild { 
            Basic.writeStreamToFile(ResourceManager.getFileAsStream(sourceName + ".tre"), new File(indexDirectory, cNameLowerCase + ".tre"));

            Basic.writeStreamToFile(ResourceManager.getFileAsStream(sourceName + ".map"), new File(indexDirectory, cNameLowerCase + ".map"));



            if (cName2SynonymsFileName.get(cName).length() > 0)

                Utilities.loadMapping(cName2SynonymsFileName.get(cName), IdMapper.MapType.Synonyms, cName);

            if (cName2AcessionFileName.get(cName).length() > 0)

                Utilities.loadMapping(cName2AcessionFileName.get(cName), IdMapper.MapType.Accession, cName);

            if (cName2GIFileName.get(cName).length() > 0)

                Utilities.loadMapping(cName2GIFileName.get(cName), IdMapper.MapType.GI, cName);

            if (class2SynonymsFile.get(cName) != null)

                Utilities.loadMapping(class2SynonymsFile.get(cName), IdMapper.MapType.Synonyms, cName);

            if (class2AccessionFile.get(cName) != null)

                Utilities.loadMapping(class2AccessionFile.get(cName), IdMapper.MapType.Accession, cName);

            if (class2GIFile.get(cName) != null)

                Utilities.loadMapping(class2GIFile.get(cName), IdMapper.MapType.GI, cName);



            final IdParser idParser = ClassificationManager.get(cName, true).getIdMapper().createIdParser();

            if (cName.equals(Classification.Taxonomy))
@@ -271,8 +272,13 @@ public class MaltBuild { 
            referencesDB.save(new File(indexDirectory, "ref.idx"), new File(indexDirectory, "ref.db"), new File(indexDirectory, "ref.inf"), saveFirstWordOfReferenceHeaderOnly);



        if (gffFiles.size() > 0) {

            GeneTableBuilder geneTableBuilder = new GeneTableBuilder();

            geneTableBuilder.buildAndSaveAnnotations(referencesDB, gffFiles, new File(indexDirectory, "annotation.idx"), new File(indexDirectory, "annotation.db"), numberOfThreads);

            // setup gene item creator, in particular accession mapping

            final GeneItemCreator creator = AAdderBuild.setupCreator(null, class2AccessionFile);



            // obtains the gene annotations:

            Map<String, ArrayList<Interval<GeneItem>>> dnaId2list = AAdderBuild.computeAnnotations(creator, gffFiles);



            AAdderBuild.saveIndex(INDEX_CREATOR, creator, indexDirectory.getPath(), dnaId2list, referencesDB.refNames());

        }

    }

}

src/malt/MaltRun.java

+6 −6
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ import malt.align.BlastStatisticsHelper; 
import malt.align.DNAScoringMatrix;

import malt.align.ProteinScoringMatrix;

import malt.data.*;

import malt.genes.GeneTableAccess;

import malt.genes.GeneItemAccessor;

import malt.io.*;

import malt.mapping.MappingManager;

import malt.util.Utilities;
@@ -196,7 +196,7 @@ public class MaltRun { 


        ReadMagnitudeParser.setEnabled(options.getOption("mag", "magnitudes", "Reads have magnitudes (to be used in taxonomic or functional analysis)", false));



        maltOptions.setContaminantsFile(ProgramProperties.getIfEnabled("enable-contaminants", options.getOption("-cf", "conFile", "File of contaminant taxa (one Id or name per line)", "")));

        maltOptions.setContaminantsFile(options.getOption("-cf", "conFile", "File of contaminant taxa (one Id or name per line)", ""));



        options.comment("Heuristics:");

        maltOptions.setMaxSeedsPerOffsetPerFrame(options.getOption("spf", "maxSeedsPerFrame", "Maximum number of seed matches per offset per read frame", maltOptions.getMaxSeedsPerOffsetPerFrame()));
@@ -299,9 +299,9 @@ public class MaltRun { 
            MappingManager.loadMappings(cNames, indexDirectory);

        }



        final GeneTableAccess geneTableAccess;

        if ((new File(indexDirectory, "annotation.idx")).exists()) {

            geneTableAccess = new GeneTableAccess(new File(indexDirectory, "annotation.idx"), new File(indexDirectory, "annotation.db"));

        final GeneItemAccessor geneTableAccess;

        if ((new File(indexDirectory, "aadd.idx")).exists()) {

            geneTableAccess = new GeneItemAccessor(new File(indexDirectory, "aadd.idx"), new File(indexDirectory, "aadd.dbx"));

            maltOptions.setParseHeaders(true);

        } else

            geneTableAccess = null;
@@ -356,7 +356,7 @@ public class MaltRun { 
                                        final String matchesOutputFile,

                                        final String alignedReadsOutputFile, final String unalignedReadsOutputFile,

                                        final ReferencesDBAccess referencesDB, final ReferencesHashTableAccess[] tables,

                                        final GeneTableAccess geneTableAccess) throws IOException, JAXBException {

                                        final GeneItemAccessor geneTableAccess) throws IOException {



        final ExecutorService executor = Executors.newFixedThreadPool(maltOptions.getNumberOfThreads());

        final CountDownLatch countDownLatch = new CountDownLatch(maltOptions.getNumberOfThreads());

src/malt/Version.java

+1 −1
Original line number Diff line number Diff line
@@ -25,5 +25,5 @@ package malt; 
 */

public class Version {

    public static final String NAME = "MALT";

    public static final String SHORT_DESCRIPTION = "MALT (version 0.4.0, built 6 Sep 2017)";

    public static final String SHORT_DESCRIPTION = "MALT (version 0.4.1, built 29 June 2018)";

}

src/malt/data/ReferencesDBBuilder.java

+35 −2
Original line number Diff line number Diff line
@@ -19,12 +19,14 @@ 
 */

package malt.data;



import jloda.util.Basic;

import jloda.util.CanceledException;

import jloda.util.ProgressPercentage;

import malt.io.FastAFileIteratorBytes;

import megan.io.OutputWriter;



import java.io.*;

import java.util.Iterator;

import java.util.List;



/**
@@ -104,6 +106,37 @@ public class ReferencesDBBuilder implements ISequenceAccessor { 
        return headers[index];

    }



    /**

     * gets an iterable over all ref names as strings

     *

     * @return iterable

     */

    public Iterable<String> refNames() {

        return new Iterable<String>() {

            @Override

            public Iterator<String> iterator() {

                return new Iterator<String>() {

                    private int i = 0;



                    @Override

                    public boolean hasNext() {

                        return i < numberOfSequences;

                    }



                    @Override

                    public String next() {

                        return Basic.getAccessionWord(headers[i++]);

                    }



                    @Override

                    public void remove() {



                    }

                };

            }

        };

    }



    /**

     * Get sequence. Index starts at 0

     *
@@ -259,9 +292,9 @@ public class ReferencesDBBuilder implements ISequenceAccessor { 
            pos++;

        byte[] add;

        if (header[pos - 1] == '|')

            add = String.format("%s%d|", tag, id).getBytes();

            add = String.format("%s%d", tag, id).getBytes();

        else

            add = String.format("|%s%d|", tag, id).getBytes();

            add = String.format("|%s%d", tag, id).getBytes();



        byte[] newHeader = new byte[header.length + add.length];

        System.arraycopy(header, 0, newHeader, 0, pos);