README.md

@@ -3,4 +3,4 @@
 # Language: Java, Bash
 # Information about documentation is in /docs/readme.txt.
 
-# Version 38.26
+# Version 38.43

a_sample_mt.sh

@@ -26,9 +26,11 @@ Processing parameters:
 None yet!
 
 Java Parameters:
--Xmx            This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom           This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx            This will set Java's memory usage, overriding autodetection.
+                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will
+                specify 200 megs. The max is typically 85% of physical memory.
+-eoom           This flag will cause the process to exit if an out-of-memory
+                exception occurs.  Requires Java 8u92+.
 -da             Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

analyzeaccession.sh

@@ -13,9 +13,11 @@ Parameters:
 lines=-1        If positive, stop after this many lines.
 
 Java Parameters:
--Xmx            This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom           This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx            This will set Java's memory usage, overriding autodetection.
+                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will
+                specify 200 megs. The max is typically 85% of physical memory.
+-eoom           This flag will cause the process to exit if an out-of-memory
+                exception occurs.  Requires Java 8u92+.
 -da             Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

analyzegenes.sh

@@ -74,7 +74,7 @@ function analyze() {
 		module load pigz
 	fi
 	local CMD="java $EA $EOOM $z $z2 -cp $CP prok.AnalyzeGenes $@"
-	echo $CMD >&2
+	#echo $CMD >&2
 	eval $CMD
 }
 

bbcms.sh

@@ -3,7 +3,7 @@
 usage(){
 echo "
 Written by Brian Bushnell
-Last modified July 24, 2018
+Last modified December 19, 2018
 
 Description:  Error corrects reads and/or filters by depth, storing
 kmer counts in a count-min sketch (a Bloom filter variant).
@@ -89,9 +89,11 @@ smooth=3        Remove spikes from kmer counts due to hash collisions.
                 
 
 Java Parameters:
--Xmx            This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom           This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx            This will set Java's memory usage, overriding autodetection.
+                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will
+                specify 200 megs. The max is typically 85% of physical memory.
+-eoom           This flag will cause the process to exit if an out-of-memory
+                exception occurs.  Requires Java 8u92+.
 -da             Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbcountunique.sh

@@ -37,9 +37,11 @@ printlastbin=f      (plb) Print a line for the final undersized bin.
 minprob=0           Ignore kmers with a probability of correctness below this (based on q-scores).
 
 Java Parameters:
--Xmx                This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom               This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
+                    The max is typically 85% of physical memory.
+-eoom               This flag will cause the process to exit if an
+                    out-of-memory exception occurs.  Requires Java 8u92+.
 -da                 Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbduk.sh

@@ -3,7 +3,7 @@
 usage(){
 echo "
 Written by Brian Bushnell
-Last modified August 29, 2018
+Last modified March 21, 2019
 
 Description:  Compares reads to the kmers in a reference dataset, optionally 
 allowing an edit distance. Splits the reads into two outputs - those that 
@@ -70,6 +70,7 @@ maxbasesout=-1      If positive, quit after writing approximately this many
                     bases to out (outu/outnonmatch).
 maxbasesoutm=-1     If positive, quit after writing approximately this many
                     bases to outm (outmatch).
+json=f              Print to screen in json format.
 
 Histogram output parameters:
 bhist=<file>        Base composition histogram by position.
@@ -80,6 +81,7 @@ bqhist=<file>       Quality histogram designed for box plots.
 lhist=<file>        Read length histogram.
 phist=<file>        Polymer length histogram.
 gchist=<file>       Read GC content histogram.
+ihist=<file>        Insert size histogram, for paired reads in mapped sam.
 gcbins=100          Number gchist bins.  Set to 'auto' to use read length.
 maxhistlen=6000     Set an upper bound for histogram lengths; higher uses 
                     more memory.  The default is 6000 for some histograms
@@ -99,6 +101,7 @@ varfile=<file>      Ignore substitution errors listed in this file when
                     CallVariants.
 vcf=<file>          Ignore substitution errors listed in this VCF file 
                     when calculating error rates.
+ignorevcfindels=t   Also ignore indels listed in the VCF.
 
 Processing parameters:
 k=27                Kmer length used for finding contaminants.  Contaminants 
@@ -141,6 +144,8 @@ recalibrate=f       (recal) Recalibrate quality scores.  Requires calibration
                     matrices generated by CalcTrueQuality.
 sam=<file,file>     If recalibration is desired, and matrices have not already
                     been generated, BBDuk will create them from the sam file.
+amino=f             Run in amino acid mode.  Some features have not been
+                    tested, but kmer-matching works fine.  Maximum k is 12.
 
 Speed and Memory parameters:
 threads=auto        (t) Set number of threads to use; default is number of 
@@ -237,6 +242,7 @@ maxgc=1             Discard reads with GC content above this.
 gcpairs=t           Use average GC of paired reads.
                     Also affects gchist.
 tossjunk=f          Discard reads with invalid characters as bases.
+swift=f             Trim Swift sequences: Trailing C/T/N R1, leading G/A/N R2.
 
 Header-parsing parameters - these require Illumina headers:
 chastityfilter=f    (cf) Discard reads with id containing ' 1:Y:' or ' 2:Y:'.
@@ -290,8 +296,8 @@ loglogbuckets=1999  Use this many buckets for counting.
 
 Java Parameters:
 
--Xmx                This will be passed to Java to set memory usage, overriding 
-                    the program's automatic memory detection. -Xmx20g will 
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will 
                     specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  
                     The max is typically 85% of physical memory.
 -eoom               This flag will cause the process to exit if an 
@@ -322,6 +328,8 @@ z2="-Xms1g"
 EA="-ea"
 EOOM=""
 set=0
+silent=0
+json=0
 
 if [ -z "$1" ] || [[ $1 == -h ]] || [[ $1 == --help ]]; then
 	usage
@@ -365,9 +373,11 @@ bbduk() {
 		module load samtools/1.4
 		module load pigz
 	fi
-	#local CMD="java -Djava.library.path=$NATIVELIBDIR $EA $z $z2 -cp $CP jgi.BBDukF $@"
-	local CMD="java $EA $z $z2 -cp $CP jgi.BBDukF $@"
+	#local CMD="java -Djava.library.path=$NATIVELIBDIR $EA $z $z2 -cp $CP jgi.BBDuk $@"
+	local CMD="java $EA $z $z2 -cp $CP jgi.BBDuk $@"
+	if [[ $silent == 0 ]] && [[ $json == 0 ]]; then
 		echo $CMD >&2
+	fi
 	eval $CMD
 }
 

bbest.sh

@@ -15,7 +15,7 @@ in=<file>             Specify a sam file (or stdin) containing mapped ests.
 out=<file>      Specify the output stats file (default is stdout).
 ref=<file>      Specify the reference file (optional).
 est=<file>      Specify the est fasta file (optional).
-fraction=<0.98>       Min fraction of bases mapped to ref to be 
+fraction=0.98   Min fraction of bases mapped to ref to be 
                 considered 'all mapped'.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbfakereads.sh

@@ -32,9 +32,11 @@ identifier=null     (id) Output read names are prefixed with this.
 addspace=t          Set to false to omit the  space before /1 and /2 of paired reads.
 
 Java Parameters:
--Xmx                This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom               This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
+                    The max is typically 85% of physical memory.
+-eoom               This flag will cause the process to exit if an
+                    out-of-memory exception occurs.  Requires Java 8u92+.
 -da                 Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbmap.sh

@@ -4,7 +4,7 @@ usage(){
 echo "
 BBMap
 Written by Brian Bushnell, from Dec. 2010 - present
-Last modified September 18, 2018
+Last modified February 11, 2019
 
 Description:  Fast and accurate splice-aware read aligner.
 Please read bbmap/docs/guides/BBMapGuide.txt for more information.
@@ -291,8 +291,8 @@ delcoverage=t           (delcov) Count bases covered by deletions as covered.
 covk=0                  If positive, calculate kmer coverage statistics.
 
 Java Parameters:
--Xmx                    This will be passed to Java to set memory usage, 
-                        overriding the program's automatic memory detection.
+-Xmx                    This will set Java's memory usage, 
+                        overriding autodetection.
                         -Xmx20g will specify 20 gigs of RAM, and -Xmx800m 
                         will specify 800 megs.  The max is typically 85% of 
                         physical memory.  The human genome requires around 24g,

bbmask.sh

@@ -54,9 +54,11 @@ pigz=t              Use pigz to compress.  If argument is a number, that will se
 unpigz=t            Use pigz to decompress.
 
 Java Parameters:
--Xmx                This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom               This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
+                    The max is typically 85% of physical memory.
+-eoom               This flag will cause the process to exit if an
+                    out-of-memory exception occurs.  Requires Java 8u92+.
 -da                 Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbmerge.sh

@@ -3,7 +3,7 @@
 usage(){
 echo "
 Written by Brian Bushnell and Jonathan Rood
-Last modified August 23, 2018
+Last modified February 21, 2019
 
 Description:  Merges paired reads into single reads by overlap detection.
 With sufficient coverage, can merge nonoverlapping reads by kmer extension.
@@ -116,10 +116,13 @@ ratiomode=t          Score overlaps based on the ratio of matching to
 maxratio=0.09        Max error rate; higher increases merge rate.
 ratiomargin=5.5      Lower increases merge rate; min is 1.
 ratiooffset=0.55     Lower increases merge rate; min is 0.
+maxmismatches=20     Maximum mismatches allowed in overlapping region.
 ratiominoverlapreduction=3  This is the difference between minoverlap in 
                      flat mode and minoverlap in ratio mode; generally, 
                      minoverlap should be lower in ratio mode.
 minsecondratio=0.1   Cutoff for second-best overlap ratio.
+forcemerge=f         Disable all filters and just merge everything 
+                     (not recommended).
 
 Flat Mode: 
 flatmode=f           Score overlaps based on the total number of mismatching
@@ -187,6 +190,8 @@ prealloc=f           Pre-allocate memory rather than dynamically growing;
 prefilter=0          If set to a positive integer, use a countmin sketch to 
                      ignore kmers with depth of that value or lower, to
                      reduce memory usage.
+filtermem=0          Allows manually specifying prefilter memory in bytes, for
+                     deterministic runs.  0 will set it automatically.
 minprob=0.5          Ignore kmers with overall probability of correctness 
                      below this, to reduce memory usage.
 minapproxoverlap=26  For rem mode, do not merge reads if the extended reads
@@ -204,8 +209,8 @@ testmerge=f          Test kmer counts around the read merge junctions.  If
                      rem or rsem.
 
 Java Parameters:
--Xmx                 This will be passed to Java to set memory usage, 
-                     overriding the program's automatic memory detection.
+-Xmx                 This will set Java's memory usage, 
+                     overriding autodetection.
                      For example, -Xmx400m will specify 400 MB RAM.
 -eoom                This flag will cause the process to exit if an 
                      out-of-memory exception occurs.  Requires Java 8u92+.

bbnorm.sh

@@ -111,9 +111,11 @@ maxPeak=BIG         (maxp) Ignore peaks with an X-value above this.
 maxPeakCount=8      (maxpc) Print up to this many peaks (prioritizing height).
 
 Java Parameters:
--Xmx                This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom               This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
+                    The max is typically 85% of physical memory.
+-eoom               This flag will cause the process to exit if an
+                    out-of-memory exception occurs.  Requires Java 8u92+.
 -da                 Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbrealign.sh

@@ -3,7 +3,7 @@
 usage(){
 echo "
 Written by Brian Bushnell
-Last modified April 26, 2017
+Last modified November 8, 2018
 
 Description:  Realigns mapped reads to a reference.
 
@@ -52,9 +52,11 @@ supplimentary=f Include supplimentary alignments.
 invert=f        Invert sam filters.
 
 Java Parameters:
--Xmx            This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom           This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx            This will set Java's memory usage, overriding autodetection.
+                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will
+                specify 200 megs. The max is typically 85% of physical memory.
+-eoom           This flag will cause the process to exit if an out-of-memory
+                exception occurs.  Requires Java 8u92+.
 -da             Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bbsplit.sh

@@ -72,9 +72,11 @@ BBSplit is recommended for fastq and fasta output, not for sam/bam output.
 When the reference sequences are shorter than read length, use Seal instead of BBSplit.
 
 Java Parameters:
--Xmx                 This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                     -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom                This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
+                    The max is typically 85% of physical memory.
+-eoom               This flag will cause the process to exit if an
+                    out-of-memory exception occurs.  Requires Java 8u92+.
 -da                 Disable assertions.
 
 This list is not complete.  For more information, please consult $DIRdocs/readme.txt

bbsplitpairs.sh

@@ -33,9 +33,11 @@ repair=f            (rp) Fixes arbitrarily corrupted paired reads by examining r
 ain=f               (allowidenticalnames) When detecting pair names, allows identical names, instead of requiring /1 and /2 or 1: and 2:
 
 Java Parameters:
--Xmx                This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom               This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx                This will set Java's memory usage, overriding autodetection.
+                    -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.
+                    The max is typically 85% of physical memory.
+-eoom               This flag will cause the process to exit if an
+                    out-of-memory exception occurs.  Requires Java 8u92+.
 -da                 Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

bloomfilter.sh

@@ -52,9 +52,11 @@ requireboth=f   Require both reads in a pair to match the ref in order to go
                 to outm.  By default, pairs go to outm if either matches.
 
 Java Parameters:
--Xmx            This will be passed to Java to set memory usage, overriding the program's automatic memory detection.
-                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will specify 200 megs.  The max is typically 85% of physical memory.
--eoom           This flag will cause the process to exit if an out-of-memory exception occurs.  Requires Java 8u92+.
+-Xmx            This will set Java's memory usage, overriding autodetection.
+                -Xmx20g will specify 20 gigs of RAM, and -Xmx200m will
+                specify 200 megs. The max is typically 85% of physical memory.
+-eoom           This flag will cause the process to exit if an out-of-memory
+                exception occurs.  Requires Java 8u92+.
 -da             Disable assertions.
 
 Please contact Brian Bushnell at bbushnell@lbl.gov if you encounter any problems.

build.xml

-<project name="bbtools" default="dist" basedir=".">
-    <description>
-        Brian Bushnell's tools!
-    </description>
-
-    <!-- genepool specific values; you can override these from the command line -->
-    <property name="jcompiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/>
-    <property name="mpijar" location="/usr/common/usg/hpc/openmpi/gnu4.6/sge/1.8.1/ib_2.1-1.0.0/lib/mpi.jar"/>
-
-    <!-- set global properties for this build -->
-    <property name="src" location="current"/>
-    <property name="build" location="build"/>
-    <property name="dist" location="dist"/>
-    <property name="resources" location="resources"/>
-
-    <path id="class.path">
-        <pathelement location="${mpijar}"/>
-    </path>
-
-    <target name="init">
-        <!-- Create the time stamp -->
-        <tstamp/>
-        <!-- Create the build directory structure used by compile -->
-        <mkdir dir="${build}"/>
-    </target>
-
-    <target name="compile" depends="init"
-        description="compile the source " >
-
-        <!-- Compile the java code from ${src} into ${build} -->
-        <javac srcdir="${src}" destdir="${build}" compiler="${jcompiler}" nowarn="true" includeantruntime="false" source="1.7" target="1.7" debug="true" debuglevel="lines,vars,source" >
-            <classpath refid="class.path" />
-            <exclude name="jgi/KmerNormalizeExact.java" />
-        </javac>
-    </target>
-
-    <target name="dist" depends="compile"
-        description="generate the distribution" >
-
-        <!-- Create the distribution directory -->
-        <mkdir dir="${dist}/lib"/>
-        <!-- Put everything in ${build} into the MyProject-${DSTAMP}.jar file -->
-        <jar jarfile="${dist}/lib/BBTools.jar">
-            <fileset dir="${build}"/>
-            <fileset dir="${resources}"/>
-        </jar>
-    </target>
-
-    <target name="clean"
-        description="clean up" >
-
-        <!-- Delete the ${build} and ${dist} directory trees -->
-        <delete dir="${build}"/>
-        <delete dir="${dist}"/>
-    </target>
-</project>

calcmem.sh

 #!/bin/bash
 
 #usage(){
-#	echo "CalcMem v1.08"
+#	echo "CalcMem v1.09"
 #	echo "Written by Brian Bushnell, Doug Jacobsen, Alex Copeland, Bryce Foster"
 #	echo "Calculates available memory in megabytes"
-#	echo "Last modified September 6, 2018"
+#	echo "Last modified December 4, 2018"
 #}
 
 #Also parses other Java flags
@@ -43,6 +43,10 @@ function parseXmx () {
 			EOOM="-XX:+ExitOnOutOfMemoryError"
 		elif [[ "$arg" == -ExitOnOutOfMemoryError ]] || [[ "$arg" == -exitonoutofmemoryerror ]] || [[ "$arg" == -eoom ]]; then
 			EOOM="-XX:+ExitOnOutOfMemoryError"
+		elif [[ "$arg" == json ]] || [[ "$arg" == "json=t" ]] || [[ "$arg" == "json=true" ]] || [[ "$arg" == "format=json" ]]; then
+			json=1
+		elif [[ "$arg" == silent ]] || [[ "$arg" == "silent=t" ]] || [[ "$arg" == "silent=true" ]]; then
+			silent=1
 		fi
 	done