Commit 4a3900a3 authored by Andreas Tille's avatar Andreas Tille
Browse files

New upstream version 0.0+git20180524.5984e06

parent 9f97f4f9

antbuild/build.xml

+1 −2
Original line number Diff line number Diff line
@@ -19,7 +19,6 @@ 
	    <fileset dir="../../jloda/jars/batik-1.8"  includes="*.jar"/>

    	    <fileset dir="../../megan-ce/jars"  includes="*.jar"/>

	    <fileset dir="../../megan-ce/jars/megan6server"  includes="*.jar"/>

	    <fileset dir="../../malt/jars/"  includes="*.jar"/>

    </path>



    <!-- init -->
@@ -62,7 +61,7 @@ 
    <target name="jar" depends="compile">

        <jar jarfile="${jar}"

             basedir="${classDir}"

             includes="jloda/** megan/** log4j.properties malt/** rusch/**">

             includes="jloda/** megan/** log4j.properties malt/**">

        </jar>

    </target>

installer/malt.install4j

+4 −4
Original line number Diff line number Diff line 
<?xml version="1.0" encoding="UTF-8"?>

<install4j version="6.1.3" transformSequenceNumber="5">

<install4j version="6.1.6" transformSequenceNumber="5">

  <directoryPresets config="../../megan6/jars/data.jar" />

  <application name="MALT" distributionSourceDir="" applicationId="3229-5251-7410-5330" mediaDir="../../../builds" mediaFilePattern="${compiler:sys.shortName}_${compiler:sys.platform}_${compiler:sys.version}" compression="6" lzmaCompression="false" pack200Compression="false" excludeSignedFromPacking="true" commonExternalFiles="false" createMd5Sums="true" shrinkRuntime="true" shortName="MALT" publisher="Daniel Huson's Lab, University of Tuebingen" publisherWeb="www-ab.informatik.uni-tuebingen.de" version="0" allPathsRelative="true" backupOnSave="true" autoSave="false" convertDotsToUnderscores="true" macSignature="????" macVolumeId="970a61501b2fa775" javaMinVersion="1.8" javaMaxVersion="" allowBetaVM="true" jdkMode="runtimeJre" jdkName="">

    <languages skipLanguageSelection="false" languageSelectionInPrincipalLanguage="false">
@@ -15,7 +15,7 @@ 
      <variable name="variable" value="" description="" category="" />

    </variables>

    <mergedProjects />

    <codeSigning macEnabled="true" macPkcs12File="../../../../etc/Certificates.p12" windowsEnabled="false" windowsKeySource="pvkAndSpc" windowsPvkFile="" windowsSpcFile="" windowsPkcs12File="" />

    <codeSigning macEnabled="true" macPkcs12File="../../../../etc/application.p12" windowsEnabled="false" windowsKeySource="pvkAndSpc" windowsPvkFile="" windowsSpcFile="" windowsPkcs12File="" />

  </application>

  <files keepModificationTimes="false" missingFilesStrategy="warn" globalExcludeSuffixes=".svn,.CVS,*.psd,*.java" defaultOverwriteMode="4" defaultUninstallMode="0" launcherOverwriteMode="3" defaultFileMode="644" defaultDirMode="755">

    <filesets />
@@ -1461,7 +1461,7 @@ wizardContext.setCancelButtonVisible(false);</preActivation> 
        <content />

      </installerScript>

    </unixInstaller>

    <macosFolder name="Mac OS X Folder" id="1691242149" customizedId="" mediaFileName="" installDir="MALT" overridePrincipalLanguage="true" jreBitType="all" runPostProcessor="false" postProcessor="" failOnPostProcessorError="false" useLegacyMediaFileIds="false" legacyMediaFileIds="" downloadURL="" includeAllDownloadableComponents="false" includedJRE="macosx-amd64-1.8.0_112" manualJREEntry="false" bundleType="1" jreURL="" jreShared="false" directDownload="false" installOnlyIfNecessary="false" appleJre="false" requiredVmIdPrefix="" customInstallBaseDir="" contentFilesType="2" installerName="${i18n:InstallerName(${compiler:sys.fullName})}" volumeName="${compiler:sys.shortName}" compressDmg="false" signLaunchers="false">

    <macosFolder name="Mac OS X Folder" id="1691242149" customizedId="" mediaFileName="" installDir="MALT" overridePrincipalLanguage="true" jreBitType="all" runPostProcessor="false" postProcessor="" failOnPostProcessorError="false" useLegacyMediaFileIds="false" legacyMediaFileIds="" downloadURL="" includeAllDownloadableComponents="false" includedJRE="macosx-amd64-1.8.0_162" manualJREEntry="false" bundleType="1" jreURL="" jreShared="false" directDownload="false" installOnlyIfNecessary="false" appleJre="false" requiredVmIdPrefix="" customInstallBaseDir="" contentFilesType="1" installerName="${i18n:InstallerName(${compiler:sys.fullName})}" volumeName="${compiler:sys.shortName}" compressDmg="false" signLaunchers="false">

      <excludedComponents />

      <includedDownloadableComponents />

      <excludedLaunchers />
@@ -1475,7 +1475,7 @@ wizardContext.setCancelButtonVisible(false);</preActivation> 
      </autoUpdate>

      <topLevelFiles />

    </macosFolder>

    <windows name="Windows" id="1691242407" customizedId="" mediaFileName="${compiler:sys.shortName}_${compiler:sys.platform}_${compiler:sys.version}" installDir="Malt" overridePrincipalLanguage="true" jreBitType="64" runPostProcessor="false" postProcessor="" failOnPostProcessorError="false" useLegacyMediaFileIds="false" legacyMediaFileIds="" downloadURL="" includeAllDownloadableComponents="false" includedJRE="windows-amd64-1.8.0_112" manualJREEntry="false" bundleType="1" jreURL="" jreShared="false" directDownload="false" installOnlyIfNecessary="true" customInstallBaseDir="" contentFilesType="1" verifyIntegrity="true">

    <windows name="Windows" id="1691242407" customizedId="" mediaFileName="${compiler:sys.shortName}_${compiler:sys.platform}_${compiler:sys.version}" installDir="Malt" overridePrincipalLanguage="true" jreBitType="64" runPostProcessor="false" postProcessor="" failOnPostProcessorError="false" useLegacyMediaFileIds="false" legacyMediaFileIds="" downloadURL="" includeAllDownloadableComponents="false" includedJRE="windows-amd64-1.8.0_162" manualJREEntry="false" bundleType="1" jreURL="" jreShared="false" directDownload="false" installOnlyIfNecessary="true" customInstallBaseDir="" contentFilesType="1" verifyIntegrity="true">

      <excludedComponents />

      <includedDownloadableComponents />

      <excludedLaunchers />

src/malt/AlignmentEngine.java

+51 −90
Original line number Diff line number Diff line 
/**

 * AlignmentEngine.java

 * Copyright (C) 2015 Daniel H. Huson

 *

 * Copyright (C) 2018 Daniel H. Huson

 * <p>

 * (Some files contain contributions from other authors, who are then mentioned separately.)

 *

 * <p>

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * <p>

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * <p>

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */
@@ -22,16 +22,14 @@ package malt; 
import jloda.util.Basic;

import malt.align.AlignerOptions;

import malt.align.BandedAligner;

import malt.analysis.OrganismsProfile;

import malt.data.*;

import malt.genes.GeneTableAccess;

import malt.io.*;

import malt.mapping.MappingManager;

import malt.util.FixedSizePriorityQueue;

import malt.util.Utilities;

import megan.parsers.blast.BlastMode;



import java.io.IOException;

import java.io.OutputStream;

import java.util.Arrays;

import java.util.BitSet;

import java.util.HashMap;
@@ -56,12 +54,13 @@ public class AlignmentEngine { 
    // io:

    private final FastAReader fastAReader;

    private final MaltOptions.MatchOutputFormat matchOutputFormat;

    private final OutputStream organismsOutStream;

    private final FileWriterRanked matchesWriter;

    private final FileWriterRanked alignedReadsWriter;

    private final FileWriterRanked unalignedReadsWriter;

    private final RMA6Writer rmaWriter;



    private final GeneTableAccess geneTableAccess;



    // parameters

    private final double minRawScore;

    private final double minBitScore;
@@ -75,14 +74,12 @@ public class AlignmentEngine { 
    // keep track of all used references:

    private final BitSet alignedReferenceIds;



    private final OrganismsProfile organismsProfile;



    // used for stats:

    long countSequencesProcessed;

    long countSequencesWithAlignments;

    long countSeedMatches;

    long countHashSeedMismatches;

    long countAlignments;

    private long countSequencesProcessed;

    private long countSequencesWithAlignments;

    private long countSeedMatches;

    private long countHashSeedMismatches;

    private long countAlignments;



    // used in inner loop:

    private final FixedSizePriorityQueue<ReadMatch> matchesQueue;
@@ -99,21 +96,11 @@ public class AlignmentEngine { 


    /**

     * construct an instance of the alignment engine. Each instance is run in a separate thread

     *

     * @param maltOptions

     * @param alignerOptions

     * @param referencesDB

     * @param tables

     * @param fastAReader

     * @param matchesWriter

     * @param alignedReadsWriter

     * @param unalignedReadsWriter

     * @throws IOException

     */

    public AlignmentEngine(final int threadNumber, final MaltOptions maltOptions, AlignerOptions alignerOptions, final ReferencesDBAccess referencesDB,

    AlignmentEngine(final int threadNumber, final MaltOptions maltOptions, AlignerOptions alignerOptions, final ReferencesDBAccess referencesDB,

                    final ReferencesHashTableAccess[] tables, final FastAReader fastAReader,

                           final FileWriterRanked matchesWriter, final RMA6Writer rmaWriter, final OutputStream organismsOutStream,

                           final FileWriterRanked alignedReadsWriter, final FileWriterRanked unalignedReadsWriter) throws IOException {

                    final FileWriterRanked matchesWriter, final RMA6Writer rmaWriter,

                    final FileWriterRanked alignedReadsWriter, final FileWriterRanked unalignedReadsWriter, final GeneTableAccess geneTableAccess) throws IOException {

        this.threadNumber = threadNumber;

        this.maltOptions = maltOptions;

        this.referencesDB = referencesDB;
@@ -122,9 +109,9 @@ public class AlignmentEngine { 
        this.matchOutputFormat = maltOptions.getMatchOutputFormat();

        this.matchesWriter = matchesWriter;

        this.rmaWriter = rmaWriter;

        this.organismsOutStream = organismsOutStream;

        this.alignedReadsWriter = alignedReadsWriter;

        this.unalignedReadsWriter = unalignedReadsWriter;

        this.geneTableAccess = geneTableAccess;



        this.shift = maltOptions.getShift();
@@ -147,7 +134,6 @@ public class AlignmentEngine { 
        xDrop = alignerOptions.getUngappedXDrop(maltOptions.getMode());

        minUngappedRawScore = alignerOptions.getUngappedMinRawScore(maltOptions.getMode());





        // data structures used in inner loop:

        matchesQueue = new FixedSizePriorityQueue<>(maltOptions.getMaxAlignmentsPerQuery(), ReadMatch.createComparator());

        recycledMatchesArray = new ReadMatch[maltOptions.getMaxAlignmentsPerQuery()];
@@ -156,19 +142,13 @@ public class AlignmentEngine { 
        for (int i = 0; i < readMatchesForRefIndex.length; i++)

            readMatchesForRefIndex[i] = new ReadMatch();



        if (organismsOutStream != null) {

            organismsProfile = new OrganismsProfile(MappingManager.getTaxonomyMapping());

            organismsProfile.setTopPercent(maltOptions.getTopPercentLCA());

        } else

            organismsProfile = null;



        seedArrays = resizeAndConstructEntries(new SeedMatchArray[0], 1000, maltOptions.getMaxSeedsPerReference());

    }



    /**

     * The main outer loop. Grabs the next input read and determines all possible seed matches. Then calls the inner loop

     */

    public void runOuterLoop() {

    void runOuterLoop() {

        try {

            final int maxFramesPerQuery = Utilities.getMaxFramesPerQuery(maltOptions.getMode(), maltOptions.isDoForward(), maltOptions.isDoReverse());
@@ -220,13 +200,8 @@ public class AlignmentEngine { 


    /**

     * run the inner loop. This tries to extend all found seed matches. If caching is used, first tries to find alignments in cache

     *

     * @param query

     * @param totalSize

     * @param dataForInnerLoop

     * @throws IOException

     */

    public void runInnerLoop(final FastARecord query, final int totalSize, final DataForInnerLoop dataForInnerLoop) throws IOException {

    private void runInnerLoop(final FastARecord query, final int totalSize, final DataForInnerLoop dataForInnerLoop) throws IOException {

        countSequencesProcessed++;



        // if cache active and query found, use the cached matches:
@@ -370,8 +345,22 @@ public class AlignmentEngine { 
                                            }



                                            if (foundPlaceToKeepThisMatch) {

                                                final byte[] referenceHeader;

                                                if (geneTableAccess == null)

                                                    referenceHeader = referencesDB.getHeader(refIndex);

                                                else {

                                                    int start = aligner.getStartReference();

                                                    if (start == -1) {

                                                        aligner.computeAlignmentByTraceBack();

                                                        start = aligner.getStartReference();

                                                    }

                                                    int end = aligner.getEndReference();

                                                    referenceHeader = geneTableAccess.annotateRefString(Basic.toString(referencesDB.getHeader(refIndex)), refIndex, start, end).getBytes();

                                                    // System.err.println(Basic.toString(referenceHeader));

                                                }



                                                byte[] text = null;

                                                byte[] rma3Text = null;

                                                byte[] rma6Text = null;

                                                if (matchesWriter != null) {

                                                    switch (matchOutputFormat) {

                                                        default:
@@ -380,21 +369,21 @@ public class AlignmentEngine { 
                                                            break;

                                                        }

                                                        case Tab: {

                                                            text = aligner.getAlignmentTab(dataForInnerLoop, null, referencesDB.getHeader(refIndex), seedMatch.getRank()); // don't pass queryHeader, it is added below

                                                            text = aligner.getAlignmentTab(dataForInnerLoop, null, referenceHeader, seedMatch.getRank()); // don't pass queryHeader, it is added below

                                                            break;

                                                        }

                                                        case SAM: {

                                                            rma3Text = text = aligner.getAlignmentSAM(dataForInnerLoop, null, query.getSequence(), referencesDB.getHeader(refIndex), seedMatch.getRank()); // don't pass queryHeader, it is added below

                                                            rma6Text = text = aligner.getAlignmentSAM(dataForInnerLoop, null, query.getSequence(), referenceHeader, seedMatch.getRank()); // don't pass queryHeader, it is added below

                                                            break;

                                                        }

                                                    }

                                                }

                                                if (rmaWriter != null && rma3Text == null) {

                                                    rma3Text = aligner.getAlignmentSAM(dataForInnerLoop, null, query.getSequence(), referencesDB.getHeader(refIndex), seedMatch.getRank()); // don't pass queryHeader, it is added below

                                                if (rmaWriter != null && rma6Text == null) {

                                                    rma6Text = aligner.getAlignmentSAM(dataForInnerLoop, null, query.getSequence(), referenceHeader, seedMatch.getRank()); // don't pass queryHeader, it is added below

                                                }

                                                if (percentIdentity > 0) // need to filter by percent identity. Can't do this earlier because number of matches not known until alignment has been computed

                                                {

                                                    if (text == null && rma3Text == null)  // haven't computed alignment, so number of matches not yet computed

                                                    if (text == null && rma6Text == null)  // haven't computed alignment, so number of matches not yet computed

                                                        aligner.computeAlignmentByTraceBack(); // compute number of matches

                                                    if (aligner.getIdentities() < percentIdentity * aligner.getAlignmentLength()) {  // too few identities

                                                        if (incrementedNumberOfReadMatchesForRefIndex)
@@ -402,7 +391,7 @@ public class AlignmentEngine { 
                                                        continue;

                                                    }

                                                }

                                                readMatch.set(aligner.getBitScore(), refIndex, text, rma3Text, aligner.getStartReference(), aligner.getEndReference());

                                                readMatch.set(aligner.getBitScore(), refIndex, text, rma6Text, aligner.getStartReference(), aligner.getEndReference());

                                            }

                                            previous = seedMatch;

                                        }
@@ -477,10 +466,6 @@ public class AlignmentEngine { 
                }

            }



            if (organismsOutStream != null) {

                organismsProfile.addRead(Utilities.getFirstWordSkipLeadingGreaterSign(query.getHeader()), numberOfMatches, matchesArray);

            }



            if (alignedReadsWriter != null) {

                alignedReadsWriter.writeByRank(threadNumber, query.getId(), Utilities.getFirstWordEnsureLeadingGreaterSign(query.getHeader()), Utilities.copy0Terminated(query.getSequence()));

            }
@@ -502,10 +487,6 @@ public class AlignmentEngine { 
            if (rmaWriter != null && maltOptions.isSaveUnalignedToRMA()) {

                rmaWriter.processMatches(query.getHeaderString(), query.getSequenceString(), matchesArray, 0);

            }

            if (organismsOutStream != null) {

                organismsProfile.addNoHitsRead();

            }



            if (alignedReadsWriter != null) {

                alignedReadsWriter.skipByRank(threadNumber, query.getId());

            }
@@ -519,18 +500,12 @@ public class AlignmentEngine { 
     * finish up after outer loop completed

     */

    public void finish() {

        if (organismsOutStream != null) {

            organismsProfile.finishAnalysis();

        }

    }



    /**

     * compute total sequences processed

     *

     * @param alignmentEngines

     * @return total

     */

    public static long getTotalSequencesProcessed(final AlignmentEngine[] alignmentEngines) {

    static long getTotalSequencesProcessed(final AlignmentEngine[] alignmentEngines) {

        long total = 0;

        for (AlignmentEngine alignmentEngine : alignmentEngines) {

            total += alignmentEngine.countSequencesProcessed;
@@ -540,11 +515,8 @@ public class AlignmentEngine { 


    /**

     * compute total with alignments

     *

     * @param alignmentEngines

     * @return total

     */

    public static long getTotalSequencesWithAlignments(final AlignmentEngine[] alignmentEngines) {

    static long getTotalSequencesWithAlignments(final AlignmentEngine[] alignmentEngines) {

        long total = 0;

        for (AlignmentEngine alignmentEngine : alignmentEngines) {

            total += alignmentEngine.countSequencesWithAlignments;
@@ -554,11 +526,8 @@ public class AlignmentEngine { 


    /**

     * compute total number of alignments

     *

     * @param alignmentEngines

     * @return total

     */

    public static long getTotalAlignments(final AlignmentEngine[] alignmentEngines) {

    static long getTotalAlignments(final AlignmentEngine[] alignmentEngines) {

        long total = 0;

        for (AlignmentEngine alignmentEngine : alignmentEngines) {

            total += alignmentEngine.countAlignments;
@@ -566,22 +535,14 @@ public class AlignmentEngine { 
        return total;

    }



    public OrganismsProfile getOrganismsProfile() {

        return organismsProfile;

    }



    public BitSet getAlignedReferenceIds() {

    BitSet getAlignedReferenceIds() {

        return alignedReferenceIds;

    }



    /**

     * resize the array of seed match arrays

     *

     * @param array

     * @param newSize

     * @return new array

     */

    public SeedMatchArray[] resizeAndConstructEntries(SeedMatchArray[] array, int newSize, int maxLength) {

    private SeedMatchArray[] resizeAndConstructEntries(SeedMatchArray[] array, int newSize, int maxLength) {

        SeedMatchArray[] result = new SeedMatchArray[newSize];

        for (int i = array.length; i < newSize; i++)

            result[i] = new SeedMatchArray(maxLength);
@@ -592,7 +553,7 @@ public class AlignmentEngine { 
    /**

     * initialize the read sequence 2 matches cache

     */

    public static void activateReplicateQueryCaching(int bits) {

    static void activateReplicateQueryCaching(int bits) {

        System.err.println("Using replicate query cache (cache size=" + (1 << bits) + ")");

        querySequence2MatchesCache = new QuerySequence2MatchesCache(bits);

    }
@@ -600,7 +561,7 @@ public class AlignmentEngine { 
    /**

     * report on cache usage, if any

     */

    public static void reportStats() {

    static void reportStats() {

        if (querySequence2MatchesCache != null)

            querySequence2MatchesCache.reportStats();

    }
@@ -634,8 +595,8 @@ public class AlignmentEngine { 
            return matches[i];

        }



        public SeedMatch setNext(int queryOffset, int referenceOffset, int rank, int seedLength) {

            return matches[size++].set(queryOffset, referenceOffset, rank, seedLength);

        void setNext(int queryOffset, int referenceOffset, int rank, int seedLength) {

            matches[size++].set(queryOffset, referenceOffset, rank, seedLength);

        }



        public void clear() {

src/malt/DataForInnerLoop.java

+6 −6
Original line number Diff line number Diff line 
/**

 * DataForInnerLoop.java

 * Copyright (C) 2015 Daniel H. Huson

 *

 * Copyright (C) 2018 Daniel H. Huson

 * <p>

 * (Some files contain contributions from other authors, who are then mentioned separately.)

 *

 * <p>

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * <p>

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * <p>

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */

src/malt/ITextProducer.java

+6 −6
Original line number Diff line number Diff line 
/**

 * ITextProducer.java

 * Copyright (C) 2015 Daniel H. Huson

 *

 * Copyright (C) 2018 Daniel H. Huson

 * <p>

 * (Some files contain contributions from other authors, who are then mentioned separately.)

 *

 * <p>

 * This program is free software: you can redistribute it and/or modify

 * it under the terms of the GNU General Public License as published by

 * the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 *

 * <p>

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * <p>

 * You should have received a copy of the GNU General Public License

 * along with this program.  If not, see <http://www.gnu.org/licenses/>.

 */