Commit cfc2c703 authored by Andreas Tille's avatar Andreas Tille
Browse files

New upstream version 2.18.2+dfsg

parent 5a9aa210

src/main/java/htsjdk/samtools/BAMFileReader.java

+17 −0
Original line number Diff line number Diff line
@@ -337,6 +337,15 @@ public class BAMFileReader extends SamReader.ReaderImplementation { 
        return offset;

    }



    /**

     * Reads through the header and sequence records to find the virtual file offset of the first record in the BAM file.

     * The caller is responsible for closing the stream.

     */

    static long findVirtualOffsetOfFirstRecord(final SeekableStream seekableStream) throws IOException {

        final BAMFileReader reader = new BAMFileReader(seekableStream, (SeekableStream) null, false, false, ValidationStringency.SILENT, new DefaultSAMRecordFactory());

        return reader.mFirstRecordPointer;

    }



    /**

     * If true, writes the source of every read into the source SAMRecords.

     * @param enabled true to write source information into each SAMRecord.
@@ -944,6 +953,14 @@ public class BAMFileReader extends SamReader.ReaderImplementation { 
        return new BAMQueryFilteringIterator(iterator, new BAMQueryMultipleIntervalsIteratorFilter(intervals, contained));

    }



    /**

     * @return a virtual file pointer for the underlying compressed stream.

     * @see BlockCompressedInputStream#getFilePointer()

     */

    public long getVirtualFilePointer() {

        return mCompressedInputStream.getFilePointer();

    }



    /**

     * Iterate over the SAMRecords defined by the sections of the file described in the ctor argument.

     */

src/main/java/htsjdk/samtools/BAMSBIIndexer.java

0 → 100644
+90 −0
Original line number Diff line number Diff line 
/*

 * The MIT License

 *

 * Copyright (c) 2018 The Broad Institute

 *

 * Permission is hereby granted, free of charge, to any person obtaining a copy

 * of this software and associated documentation files (the "Software"), to deal

 * in the Software without restriction, including without limitation the rights

 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell

 * copies of the Software, and to permit persons to whom the Software is

 * furnished to do so, subject to the following conditions:

 *

 * The above copyright notice and this permission notice shall be included in

 * all copies or substantial portions of the Software.

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER

 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,

 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

 * THE SOFTWARE.

 */

package htsjdk.samtools;



import htsjdk.samtools.cram.io.InputStreamUtils;

import htsjdk.samtools.seekablestream.SeekablePathStream;

import htsjdk.samtools.seekablestream.SeekableStream;

import htsjdk.samtools.util.BlockCompressedInputStream;

import htsjdk.samtools.util.IOUtil;



import java.io.EOFException;

import java.io.IOException;

import java.io.OutputStream;

import java.nio.ByteBuffer;

import java.nio.ByteOrder;

import java.nio.file.Files;

import java.nio.file.Path;



/**

 * Writes SBI files for BAM files, as understood by {@link SBIIndex}.

 */

public final class BAMSBIIndexer {



    /**

     * Perform indexing on the given BAM file, at the granularity level specified.

     *

     * @param bamFile the path to the BAM file

     * @param granularity write the offset of every n-th alignment to the index

     * @throws IOException as per java IO contract

     */

    public static void createIndex(final Path bamFile, final long granularity) throws IOException {

        final Path splittingBaiFile = IOUtil.addExtension(bamFile, SBIIndex.FILE_EXTENSION);

        try (SeekableStream in = new SeekablePathStream(bamFile); OutputStream out = Files.newOutputStream(splittingBaiFile)) {

            createIndex(in, out, granularity);

        }

    }



    /**

     * Perform indexing on the given BAM file, at the granularity level specified.

     *

     * @param in a seekable stream for reading the BAM file from

     * @param out the stream to write the index to

     * @param granularity write the offset of every n-th alignment to the index

     * @throws IOException as per java IO contract

     */

    public static void createIndex(final SeekableStream in, final OutputStream out, final long granularity) throws IOException {

        long recordStart = SAMUtils.findVirtualOffsetOfFirstRecordInBam(in);

        try (BlockCompressedInputStream blockIn = new BlockCompressedInputStream(in)) {

            blockIn.seek(recordStart);

            // Create a buffer for reading the BAM record lengths. BAM is little-endian.

            final ByteBuffer byteBuffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN);

            final SBIIndexWriter indexWriter = new SBIIndexWriter(out, granularity);

            while (true) {

                try {

                    recordStart = blockIn.getFilePointer();

                    // Read the length of the remainder of the BAM record (`block_size` in the SAM spec)

                    InputStreamUtils.readFully(blockIn, byteBuffer.array(), 0, 4);

                    final int blockSize = byteBuffer.getInt(0);

                    // Process the record start position, then skip to the start of the next BAM record

                    indexWriter.processRecord(recordStart);

                    InputStreamUtils.skipFully(blockIn, blockSize);

                } catch (EOFException e) {

                    break;

                }

            }

            indexWriter.finish(recordStart, in.length());

        }

    }

}

src/main/java/htsjdk/samtools/CRAMIterator.java

+27 −23
Original line number Diff line number Diff line
@@ -41,18 +41,18 @@ import htsjdk.samtools.cram.CRAMException; 
public class CRAMIterator implements SAMRecordIterator {

    private static final Log log = Log.getInstance(CRAMIterator.class);

    private final CountingInputStream countingInputStream;

    private CramHeader cramHeader;

    private ArrayList<SAMRecord> records;

    private final CramHeader cramHeader;

    private final ArrayList<SAMRecord> records;

    private SAMRecord nextRecord = null;

    private CramNormalizer normalizer;

    private final CramNormalizer normalizer;

    private byte[] refs;

    private int prevSeqId = SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX;

    public Container container;

    private SamReader mReader;

    long firstContainerOffset = 0;

    private Iterator<Container> containerIterator;

    private final Iterator<Container> containerIterator;



    private ContainerParser parser;

    private final ContainerParser parser;

    private final CRAMReferenceSource referenceSource;



    private Iterator<SAMRecord> iterator = Collections.<SAMRecord>emptyList().iterator();
@@ -68,6 +68,10 @@ public class CRAMIterator implements SAMRecordIterator { 
        this.validationStringency = validationStringency;

    }



    /**

     * `samRecordIndex` only used when validation is not `SILENT`

     * (for identification by the validator which records are invalid)

     */

    private long samRecordIndex;

    private ArrayList<CramCompressionRecord> cramRecords;
@@ -84,7 +88,7 @@ public class CRAMIterator implements SAMRecordIterator { 
        this.containerIterator = containerIterator;



        firstContainerOffset = this.countingInputStream.getCount();

        records = new ArrayList<SAMRecord>(10000);

        records = new ArrayList<SAMRecord>(CRAMContainerStreamWriter.DEFAULT_RECORDS_PER_SLICE);

        normalizer = new CramNormalizer(cramHeader.getSamFileHeader(),

                referenceSource);

        parser = new ContainerParser(cramHeader.getSamFileHeader());
@@ -103,7 +107,7 @@ public class CRAMIterator implements SAMRecordIterator { 
        this.containerIterator = containerIterator;



        firstContainerOffset = containerIterator.getFirstContainerOffset();

        records = new ArrayList<SAMRecord>(10000);

        records = new ArrayList<SAMRecord>(CRAMContainerStreamWriter.DEFAULT_RECORDS_PER_SLICE);

        normalizer = new CramNormalizer(cramHeader.getSamFileHeader(),

                referenceSource);

        parser = new ContainerParser(cramHeader.getSamFileHeader());
@@ -143,9 +147,6 @@ public class CRAMIterator implements SAMRecordIterator { 
            }

        }



        if (records == null)

            records = new ArrayList<SAMRecord>(container.nofRecords);

        else

        records.clear();

        if (cramRecords == null)

            cramRecords = new ArrayList<CramCompressionRecord>(container.nofRecords);
@@ -172,8 +173,10 @@ public class CRAMIterator implements SAMRecordIterator { 


        for (int i = 0; i < container.slices.length; i++) {

            final Slice slice = container.slices[i];



            if (slice.sequenceId < 0)

                continue;



            if (!slice.validateRefMD5(refs)) {

                final String msg = String.format(

                        "Reference sequence MD5 mismatch for slice: sequence id %d, start %d, span %d, expected MD5 %s",
@@ -201,12 +204,6 @@ public class CRAMIterator implements SAMRecordIterator { 


            samRecord.setValidationStringency(validationStringency);



            if (validationStringency != ValidationStringency.SILENT) {

                final List<SAMValidationError> validationErrors = samRecord.isValid();

                SAMUtils.processValidationErrors(validationErrors,

                        samRecordIndex, validationStringency);

            }



            if (mReader != null) {

                final long chunkStart = (container.offset << 16) | cramRecord.sliceIndex;

                final long chunkEnd = ((container.offset << 16) | cramRecord.sliceIndex) + 1;
@@ -215,7 +212,6 @@ public class CRAMIterator implements SAMRecordIterator { 
            }



            records.add(samRecord);

            samRecordIndex++;

        }

        cramRecords.clear();

        iterator = records.iterator();
@@ -267,7 +263,15 @@ public class CRAMIterator implements SAMRecordIterator { 
    @Override

    public SAMRecord next() {

        if (hasNext()) {

            return iterator.next();



            SAMRecord samRecord = iterator.next();



            if (validationStringency != ValidationStringency.SILENT) {

                SAMUtils.processValidationErrors(samRecord.isValid(), samRecordIndex++, validationStringency);

            }



            return samRecord;



        } else {

            throw new NoSuchElementException();

        }

src/main/java/htsjdk/samtools/SAMSequenceRecord.java

+16 −13
Original line number Diff line number Diff line
@@ -24,6 +24,8 @@ 
package htsjdk.samtools;





import htsjdk.variant.variantcontext.VariantContext;



import java.math.BigInteger;

import java.net.URI;

import java.net.URISyntaxException;
@@ -57,23 +59,27 @@ public class SAMSequenceRecord extends AbstractSAMHeaderRecord implements Clonea 




    /**

     * This is not a valid sequence name, because it is reserved in the MRNM field of SAM text format

     * This is not a valid sequence name, because it is reserved in the RNEXT field of SAM text format

     * to mean "same reference as RNAME field."

     */

    public static final String RESERVED_MRNM_SEQUENCE_NAME = "=";



    public static final String RESERVED_RNEXT_SEQUENCE_NAME = "=";



    /* use RESERVED_RNEXT_SEQUENCE_NAME instead. */

    @Deprecated

    public static final String RESERVED_MRNM_SEQUENCE_NAME = RESERVED_RNEXT_SEQUENCE_NAME;



    /**

     * The standard tags are stored in text header without type information, because the type of these tags is known.

     */

    public static final Set<String> STANDARD_TAGS =

            new HashSet<String>(Arrays.asList(SEQUENCE_NAME_TAG, SEQUENCE_LENGTH_TAG, ASSEMBLY_TAG, MD5_TAG, URI_TAG,

                                                SPECIES_TAG));

            new HashSet<>(Arrays.asList(SEQUENCE_NAME_TAG, SEQUENCE_LENGTH_TAG, ASSEMBLY_TAG, MD5_TAG, URI_TAG, SPECIES_TAG));



    // Split on any whitespace

    private static final Pattern SEQUENCE_NAME_SPLITTER = Pattern.compile("\\s");

    // These are the chars matched by \\s.

    private static final char[] WHITESPACE_CHARS = {' ', '\t', '\n', '\013', '\f', '\r'}; // \013 is vertical tab



    private static final Pattern LEGAL_RNAME_PATTERN = Pattern.compile("[0-9A-Za-z!#$%&+./:;?@^_|~-][0-9A-Za-z!#$%&*+./:;=?@^_|~-]*");



    /**

     * @deprecated Use {@link #SAMSequenceRecord(String, int)} instead.

     * sequenceLength is required for the object to be considered valid.
@@ -85,9 +91,6 @@ public class SAMSequenceRecord extends AbstractSAMHeaderRecord implements Clonea 


    public SAMSequenceRecord(final String name, final int sequenceLength) {

        if (name != null) {

            if (SEQUENCE_NAME_SPLITTER.matcher(name).find()) {

                throw new SAMException("Sequence name contains invalid character: " + name);

            }

            validateSequenceName(name);

            mSequenceName = name.intern();

        } else {
@@ -188,8 +191,8 @@ public class SAMSequenceRecord extends AbstractSAMHeaderRecord implements Clonea 
    public static String truncateSequenceName(final String sequenceName) {

        /*

         * Instead of using regex split, do it manually for better performance.

        return SEQUENCE_NAME_SPLITTER.split(sequenceName, 2)[0];

         */



        int truncateAt = sequenceName.length();

        for (final char c : WHITESPACE_CHARS) {

            int index = sequenceName.indexOf(c);
@@ -204,8 +207,8 @@ public class SAMSequenceRecord extends AbstractSAMHeaderRecord implements Clonea 
     * Throw an exception if the sequence name is not valid.

     */

    public static void validateSequenceName(final String name) {

        if (RESERVED_MRNM_SEQUENCE_NAME.equals(name)) {

            throw new SAMException("'" + RESERVED_MRNM_SEQUENCE_NAME + "' is not a valid sequence name");

        if (!LEGAL_RNAME_PATTERN.matcher(name).useAnchoringBounds(true).matches()) {

            throw new SAMException(String.format("Sequence name '%s' doesn't match regex: '%s' ", name, LEGAL_RNAME_PATTERN));

        }

    }

src/main/java/htsjdk/samtools/SAMUtils.java

+13 −0
Original line number Diff line number Diff line
@@ -23,6 +23,7 @@ 
 */

package htsjdk.samtools;



import htsjdk.samtools.seekablestream.SeekableStream;

import htsjdk.samtools.util.BinaryCodec;

import htsjdk.samtools.util.CigarUtil;

import htsjdk.samtools.util.CloserUtil;
@@ -685,6 +686,18 @@ public final class SAMUtils { 
        }

    }



    /**

     * Returns the virtual file offset of the first record in a BAM file - i.e. the virtual file

     * offset after skipping over the text header and the sequence records.

     */

    public static long findVirtualOffsetOfFirstRecordInBam(final SeekableStream seekableStream) {

        try {

            return BAMFileReader.findVirtualOffsetOfFirstRecord(seekableStream);

        } catch (final IOException ioe) {

            throw new RuntimeEOFException(ioe);

        }

    }



    /**

     * Given a Cigar, Returns blocks of the sequence that have been aligned directly to the

     * reference sequence. Note that clipped portions, and inserted and deleted bases (vs. the reference)