CHANGES.md

+STAR 2.6.1c 2018/11/16
+======================
+
+* Fixed the problem causing BAM sorting error with large number of threads and small ulimit -n (github.com/alexdobin/STAR/issues/512).
+* Fixed the bug causing inconsistent output for mate1/2 in the Unmapped files (github.com/alexdobin/STAR/issues/222).
+* Fixed the non-thread safe error/exit (github.com/alexdobin/STAR/issues/514), and non-safe file size check (github.com/alexdobin/STAR/issues/516)
+* Many thanks to Paul Menzel for helping to track and fix these problems.
+
+
 STAR 2.6.1c 2018/10/17
 ======================
 
 * Enforced the consistent choice of supplementary chimeric alignments for overlapping mates.
 
+
 STAR 2.6.1b 2018/09/06
 ======================
 

debian/changelog

+rna-star (2.6.1d+dfsg-1) unstable; urgency=medium
+
+  * New upstream release.
+  * Remove patch applied upstream.
+
+ -- Sascha Steinbiss <satta@debian.org>  Sat, 22 Dec 2018 21:03:02 +0000
+
 rna-star (2.6.1c+dfsg-1) unstable; urgency=medium
 
   * New upstream version.

debian/patches/series

 donotuse_own_htslib.patch
 mips_shm_noreserve.patch
 reproducible.patch
-spelling.patch

debian/patches/spelling.patch

-Description: fix spelling
-Author: Sascha Steinbiss <satta@debian.org>
-Forwarded: https://github.com/alexdobin/STAR/pull/475
---- a/extras/doc-latex/parametersDefault.tex
-+++ b/extras/doc-latex/parametersDefault.tex
-@@ -715,7 +715,7 @@
-   \optLine{int{\textgreater}=0: the score range for multi-mapping chimeras below the best chimeric score. Only works with --chimMultimapNmax {\textgreater} 1} 
- \optName{chimNonchimScoreDropMin}
-   \optValue{20}
--  \optLine{int{\textgreater}=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read lenght has to be smaller than this value} 
-+  \optLine{int{\textgreater}=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read length has to be smaller than this value} 
- \optName{chimOutJunctionFormat}
-   \optValue{0}
-   \optLine{int: formatting type for the Chimeric.out.junction file} 
---- a/source/Genome.cpp
-+++ b/source/Genome.cpp
-@@ -547,7 +547,7 @@
-         P.winBinNbits = (uint) floor( log2( max( max(4LLU,P.alignIntronMax), (P.alignMatesGapMax==0 ? 1000LLU : P.alignMatesGapMax) ) /4 ) + 0.5);
-         P.winBinNbits = max( P.winBinNbits, (uint) floor(log2(nGenome/40000+1)+0.5) ); 
-         //ISSUE - to be fixed in STAR3: if alignIntronMax>0 but alignMatesGapMax==0, winBinNbits will be defined by alignIntronMax
--        P.inOut->logMain << "To accomodate alignIntronMax="<<P.alignIntronMax<<" redefined winBinNbits="<< P.winBinNbits <<endl;
-+        P.inOut->logMain << "To accommodate alignIntronMax="<<P.alignIntronMax<<" redefined winBinNbits="<< P.winBinNbits <<endl;
- 
-     };
- 
---- a/source/ReadAlign_stitchWindowSeeds.cpp
-+++ b/source/ReadAlign_stitchWindowSeeds.cpp
-@@ -255,7 +255,7 @@
- //         };
- 
- 
--        //check exons lenghts including repeats, do not report a transcript with short exons
-+        //check exons lengths including repeats, do not report a transcript with short exons
- //        for (uint isj=0;isj<trA.nExons-1;isj++) {//check exons for min length, if they precede a junction
- //            if ( trA.canonSJ[isj]>=0 &&
- //               ( trA.exons[isj][EX_L] < P.alignSJoverhangMin + trA.shiftSJ[isj][0]
---- a/source/parametersDefault
-+++ b/source/parametersDefault
-@@ -619,7 +619,7 @@
-     int>=0: the score range for multi-mapping chimeras below the best chimeric score. Only works with --chimMultimapNmax > 1
- 
- chimNonchimScoreDropMin         20
--    int>=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read lenght has to be smaller than this value
-+    int>=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read length has to be smaller than this value
- 
- chimOutJunctionFormat           0
-     int: formatting type for the Chimeric.out.junction file
---- a/source/stitchWindowAligns.cpp
-+++ b/source/stitchWindowAligns.cpp
-@@ -93,7 +93,7 @@
-         };
-         trA.gLength = tG2+1-trA.gStart;
- 
--        //check exons lenghts including repeats, do not report a transcript with short exons
-+        //check exons lengths including repeats, do not report a transcript with short exons
-         for (uint isj=0;isj<trA.nExons-1;isj++) {//check exons for min length, if they are not annotated and precede a junction
-             if ( trA.canonSJ[isj]>=0 ) {//junction
-                 if (trA.sjAnnot[isj]==1) {//sjdb

extras/doc-latex/parametersDefault.tex

@@ -715,7 +715,7 @@
   \optLine{int{\textgreater}=0: the score range for multi-mapping chimeras below the best chimeric score. Only works with --chimMultimapNmax {\textgreater} 1} 
 \optName{chimNonchimScoreDropMin}
   \optValue{20}
-  \optLine{int{\textgreater}=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read lenght has to be smaller than this value} 
+  \optLine{int{\textgreater}=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read length has to be smaller than this value}
 \optName{chimOutJunctionFormat}
   \optValue{0}
   \optLine{int: formatting type for the Chimeric.out.junction file} 

source/BAMbinSortByCoordinate.cpp

@@ -16,11 +16,14 @@ void BAMbinSortByCoordinate(uint32 iBin, uint binN, uint binS, uint nThreads, st
         string bamInFile=dirBAMsort+to_string(it)+"/"+to_string((uint) iBin);
         ifstream bamInStream;
         bamInStream.open(bamInFile.c_str(),std::ios::binary | std::ios::ate);//open at the end to get file size
-        uint s1=bamInStream.tellg();
-        if (s1>0)
-        {
+        int64 s1=bamInStream.tellg();
+        if (s1>0)         {
             bamInStream.seekg(std::ios::beg);
             bamInStream.read(bamIn+bamInBytes,s1);//read the whole file
+        } else if (s1<0) {
+            ostringstream errOut;
+            errOut << "EXITING because of FATAL ERROR: failed reading from temporary file: " << dirBAMsort+to_string(it)+"/"+to_string((uint) iBin);
+            exitWithError(errOut.str(),std::cerr, P.inOut->logMain, 1, P);
         };
         bamInBytes += bamInStream.gcount();
         bamInStream.close();
@@ -29,7 +32,7 @@ void BAMbinSortByCoordinate(uint32 iBin, uint binN, uint binS, uint nThreads, st
     if (bamInBytes!=binS) {
         ostringstream errOut;
         errOut << "EXITING because of FATAL ERROR: number of bytes expected from the BAM bin does not agree with the actual size on disk: ";
-        errOut << binS <<"   "<< bamInBytes <<"   "<< iBin <<"\n";
+        errOut << "Expected bin size=" <<binS <<" ; size on disk="<< bamInBytes <<" ; bin number="<< iBin <<"\n";
         exitWithError(errOut.str(),std::cerr, P.inOut->logMain, 1, P);
     };
 

source/BAMoutput.cpp

@@ -4,6 +4,7 @@
 #include <pthread.h>
 #include "serviceFuns.cpp"
 #include "ThreadControl.h"
+#include "streamFuns.h"
 
 BAMoutput::BAMoutput (int iChunk, string tmpDir, Parameters &Pin) : P(Pin){//allocate bam array
 
@@ -23,7 +24,7 @@ BAMoutput::BAMoutput (int iChunk, string tmpDir, Parameters &Pin) : P(Pin){//all
     for (uint ii=0;ii<nBins;ii++) {
         binStart[ii]=bamArray+bamArraySize/nBins*ii;
         binBytes[ii]=0;
-        binStream[ii]=new ofstream((bamDir +"/"+to_string(ii)).c_str());    //open temporary files
+        binStream[ii]=&ofstrOpen((bamDir +"/"+to_string(ii)).c_str(), ERROR_OUT, P);    //open temporary files
         binTotalN[ii]=0;
         binTotalBytes[ii]=0;
     };

source/ErrorWarning.cpp

@@ -3,10 +3,12 @@ functions that handle errors and warnings
 */
 #include "ErrorWarning.h"
 #include "TimeFunctions.h"
+#include "GlobalVariables.h"
 
 void exitWithError(string messageOut, ostream &streamOut1, ostream &streamOut2, int errorInt, Parameters &P) {
     time_t timeCurrent;
     time( &timeCurrent);
+    if (P.runThreadN>1) pthread_mutex_lock(&g_threadChunks.mutexError);
     if (streamOut1.good()) {
         streamOut1 << "\n" << messageOut << endl << timeMonthDayTime(timeCurrent) <<" ...... FATAL ERROR, exiting\n"  <<flush;
     };
@@ -14,6 +16,6 @@ void exitWithError(string messageOut, ostream &streamOut1, ostream &streamOut2,
         streamOut2 << "\n" << messageOut << endl << timeMonthDayTime(timeCurrent) <<" ...... FATAL ERROR, exiting\n"  <<flush;
     };
     delete P.inOut; //to close files
-
+//     if (P.runThreadN>1) pthread_mutex_unlock(&g_threadChunks.mutexError);
     exit(errorInt);
 };

source/Genome.cpp

@@ -25,7 +25,7 @@
 //arbitrary number for ftok function
 #define SHM_projectID 23
 
-Genome::Genome (Parameters &Pin ): pGe(Pin.pGe), P(Pin), shmStart(NULL), sharedMemory(NULL) {
+Genome::Genome (Parameters &Pin ): pGe(Pin.pGe), sharedMemory(NULL), P(Pin), shmStart(NULL) {
     shmKey=ftok(pGe.gDir.c_str(),SHM_projectID);    
     
     sjdbOverhang = pGe.sjdbOverhang; //will be re-defined later if another value was used for the generated genome
@@ -55,20 +55,25 @@ uint Genome::OpenStream(string name, ifstream & stream, uint size)
     stream.open((pGe.gDir+ "/" +name).c_str(), ios::binary);
     if (!stream.good()) {
         ostringstream errOut;
-        errOut << "EXITING because of FATAL ERROR: could not open genome file "<< pGe.gDir << "/" << name <<"\n" << endl;
+        errOut << "EXITING because of FATAL ERROR: could not open genome file: "<< pGe.gDir << "/" << name <<"\n";
         errOut << "SOLUTION: check that the path to genome files, specified in --genomeDir is correct and the files are present, and have user read permissions\n" <<flush;
         exitWithError(errOut.str(),std::cerr, P.inOut->logMain, EXIT_CODE_GENOME_FILES, P);
     };
 
 
-    if (size>0)
-    {
+    if (size>0) {
         P.inOut->logMain << name << ": size given as a parameter = " << size <<"\n";
-    } else
-    {
+    } else {
         P.inOut->logMain << "Checking " << name << " size";
         stream.seekg (0, ios::end);
-        size=(uint) stream.tellg();
+        int64 size1 = stream.tellg();
+        if (size1<=0) {
+            ostringstream errOut;
+            errOut << "EXITING because of FATAL ERROR: failed reading from genome file: "<< pGe.gDir << "/" << name <<"\n";
+            errOut << "SOLUTION: re-generate the genome index\n";
+            exitWithError(errOut.str(),std::cerr, P.inOut->logMain, 1, P);
+        };
+        size=(uint) size1;
         stream.clear();
         stream.seekg (0, ios::beg);
         P.inOut->logMain << "file size: "<< size <<" bytes; state: good=" <<stream.good()\
@@ -547,7 +552,7 @@ void Genome::genomeLoad(){//allocate and load Genome
         P.winBinNbits = (uint) floor( log2( max( max(4LLU,P.alignIntronMax), (P.alignMatesGapMax==0 ? 1000LLU : P.alignMatesGapMax) ) /4 ) + 0.5);
         P.winBinNbits = max( P.winBinNbits, (uint) floor(log2(nGenome/40000+1)+0.5) ); 
         //ISSUE - to be fixed in STAR3: if alignIntronMax>0 but alignMatesGapMax==0, winBinNbits will be defined by alignIntronMax
-        P.inOut->logMain << "To accomodate alignIntronMax="<<P.alignIntronMax<<" redefined winBinNbits="<< P.winBinNbits <<endl;
+        P.inOut->logMain << "To accommodate alignIntronMax="<<P.alignIntronMax<<" redefined winBinNbits="<< P.winBinNbits <<endl;
 
     };
 

source/ReadAlignChunk_processChunks.cpp

@@ -219,9 +219,15 @@ void ReadAlignChunk::processChunks() {//read-map-write chunks
             chunkFstreamCat (*RA->chunkOutChimJunction, P.inOut->outChimJunction, P.runThreadN>1, g_threadChunks.mutexOutChimJunction);
         };
         if (P.outReadsUnmapped=="Fastx" ) {
+            if (P.runThreadN>1) 
+                pthread_mutex_lock(&g_threadChunks.mutexOutUnmappedFastx);
+
             for (uint ii=0;ii<P.readNmates;ii++) {
-                chunkFstreamCat (RA->chunkOutUnmappedReadsStream[ii],P.inOut->outUnmappedReadsStream[ii], P.runThreadN>1, g_threadChunks.mutexOutUnmappedFastx);
+                chunkFstreamCat (RA->chunkOutUnmappedReadsStream[ii],P.inOut->outUnmappedReadsStream[ii], false, g_threadChunks.mutexOutUnmappedFastx);
             };
+
+            if (P.runThreadN>1) 
+                pthread_mutex_unlock(&g_threadChunks.mutexOutUnmappedFastx);
         };
     };
     if (P.runThreadN>1) pthread_mutex_lock(&g_threadChunks.mutexLogMain);

source/ReadAlign_stitchWindowSeeds.cpp

@@ -255,7 +255,7 @@ void ReadAlign::stitchWindowSeeds (uint iW, uint iWrec, bool *WAexcl, char *R) {
 //         };
 
 
-        //check exons lenghts including repeats, do not report a transcript with short exons
+        //check exons lengths including repeats, do not report a transcript with short exons
 //        for (uint isj=0;isj<trA.nExons-1;isj++) {//check exons for min length, if they precede a junction
 //            if ( trA.canonSJ[isj]>=0 &&
 //               ( trA.exons[isj][EX_L] < P.alignSJoverhangMin + trA.shiftSJ[isj][0]

source/STAR.cpp

@@ -133,6 +133,7 @@ int main(int argInN, char* argIn[]) {
         pthread_mutex_init(&g_threadChunks.mutexOutFilterBySJout, NULL);
         pthread_mutex_init(&g_threadChunks.mutexStats, NULL);
         pthread_mutex_init(&g_threadChunks.mutexBAMsortBins, NULL);
+        pthread_mutex_init(&g_threadChunks.mutexError, NULL);
     };
 
     g_statsAll.progressReportHeader(P.inOut->logProgress);

source/ThreadControl.h

@@ -12,7 +12,7 @@ public:
 
     pthread_t *threadArray;
     pthread_mutex_t mutexInRead, mutexOutSAM, mutexOutBAM1, mutexOutChimSAM, mutexOutChimJunction, mutexOutUnmappedFastx, mutexOutFilterBySJout;
-    pthread_mutex_t mutexStats, mutexLogMain, mutexBAMsortBins;
+    pthread_mutex_t mutexStats, mutexLogMain, mutexBAMsortBins, mutexError;
 
     uint chunkInN,chunkOutN;
 

source/VERSION

-#define STAR_VERSION "STAR_2.6.1c"
+#define STAR_VERSION "STAR_2.6.1d"

source/Variation.cpp

@@ -56,7 +56,10 @@ void scanVCF(ifstream& vcf, Parameters& P, SNP& snp, vector <uint> &chrStart, ma
                     //this is a strange case in VCF when ALT allele(s) are equal to REF
                 } else {
                     snp.lociV.push_back(pos-1+chrStart[chrNameIndex[chr]]);
-                    array<char,3> nt1={convertNt01234( ref.at(0) ), convertNt01234( altV.at( atoi(&sample.at(0)) ).at(0) ), convertNt01234( altV.at( atoi(&sample.at(2)) ).at(0) )};
+                    array<char,3> nt1;
+                    nt1[0]=convertNt01234( ref.at(0) );
+                    nt1[1]=convertNt01234( altV.at( atoi(&sample.at(0)) ).at(0) );
+                    nt1[2]=convertNt01234( altV.at( atoi(&sample.at(2)) ).at(0) );
                     snp.nt.push_back(nt1);
                     snp.N++;
                 };

source/parametersDefault

@@ -619,7 +619,7 @@ chimMultimapScoreRange          1
     int>=0: the score range for multi-mapping chimeras below the best chimeric score. Only works with --chimMultimapNmax > 1
 
 chimNonchimScoreDropMin         20
-    int>=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read lenght has to be smaller than this value
+    int>=0: to trigger chimeric detection, the drop in the best non-chimeric alignment score with respect to the read length has to be smaller than this value
 
 chimOutJunctionFormat           0
     int: formatting type for the Chimeric.out.junction file

source/parametersDefault.xxd

@@ -2771,7 +2771,7 @@ unsigned char parametersDefault[] = {
   0x20, 0x61, 0x6c, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x20, 0x73,
   0x63, 0x6f, 0x72, 0x65, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x72, 0x65,
   0x73, 0x70, 0x65, 0x63, 0x74, 0x20, 0x74, 0x6f, 0x20, 0x74, 0x68, 0x65,
-  0x20, 0x72, 0x65, 0x61, 0x64, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x68, 0x74,
+  0x20, 0x72, 0x65, 0x61, 0x64, 0x20, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68,
   0x20, 0x68, 0x61, 0x73, 0x20, 0x74, 0x6f, 0x20, 0x62, 0x65, 0x20, 0x73,
   0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x20, 0x74, 0x68, 0x61, 0x6e, 0x20,
   0x74, 0x68, 0x69, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x0a, 0x0a,

source/readLoad.cpp

@@ -37,7 +37,7 @@ int readLoad(istream& readInStream, Parameters& P, uint iMate, uint& Lread, uint
     readInStream.getline(Seq,DEF_readSeqLengthMax+1); //extract sequence
 
     Lread=0;
-    for (uint ii=0; ii<readInStream.gcount()-1; ii++) {
+    for (int ii=0; ii<readInStream.gcount()-1; ii++) {
         if (int(Seq[ii])>=32) {
             Seq[Lread]=Seq[ii];
             ++Lread;

source/stitchWindowAligns.cpp

@@ -93,7 +93,7 @@ void stitchWindowAligns(uint iA, uint nA, int Score, bool WAincl[], uint tR2, ui
         };
         trA.gLength = tG2+1-trA.gStart;
 
-        //check exons lenghts including repeats, do not report a transcript with short exons
+        //check exons lengths including repeats, do not report a transcript with short exons
         for (uint isj=0;isj<trA.nExons-1;isj++) {//check exons for min length, if they are not annotated and precede a junction
             if ( trA.canonSJ[isj]>=0 ) {//junction
                 if (trA.sjAnnot[isj]==1) {//sjdb

source/streamFuns.cpp

@@ -61,11 +61,9 @@ void fstreamWriteBig(std::ofstream &S, char* A, unsigned long long N, std::strin
 std::ofstream &ofstrOpen (std::string fileName, std::string errorID, Parameters &P) {//open file 'fileName', generate error if cannot open
     std::ofstream & ofStream = *new std::ofstream(fileName.c_str(), std::fstream::out | std::fstream::trunc);
     if (ofStream.fail()) {//
-//         dir1=fileName.substr(0,fileName.find_last_of("/")+1);
-//         if (dir1=="") dir1="./";
         ostringstream errOut;
         errOut << errorID<<": exiting because of *OUTPUT FILE* error: could not create output file "<< fileName <<"\n";
-        errOut << "Solution: check that the path exists and you have write permission for this file\n";
+        errOut << "SOLUTION: check that the path exists and you have write permission for this file. Also check ""ulimit -n"" and increase it to allow more open files.\n";
         exitWithError(errOut.str(),std::cerr, P.inOut->logMain, EXIT_CODE_FILE_OPEN, P);
     };
     return ofStream;