Loading .gitignore +2 −0 Original line number Diff line number Diff line *.pyc *.idx *.swo *.swp bcbio/pipeline/version.py bcbio_nextgen.egg-info/ build/ Loading HISTORY.md +21 −0 Original line number Diff line number Diff line ## 1.1.4 (3 April 2019) - Move to Python 3.6. A python2 environment in the install runs non python3 compatible programs. The codebase is still compatible with python 2.7 but will only get run and tested on python 3 for future releases. - RNA-seq: fix for race condition when creating the pizzly cache - RNA-seq: Add Salmon to multiqc report. - RNA-seq single-cell/DGE: Properly strip transcript versions from GENCODE GTFs. - RNA-seq: Faster and more flexible rRNA biotype lookup. - Move to R3.5.1, including updates to all CRAN and Bioconductor packages. - tumor-only germline prioritization: provide more useful germline filtering based on prioritization INFO tag (EPR) rather than filter field. - Install: do not require fabric for tool and data installs, making full codebase compatible with python 3. - variant: Filter out variants with missing ALT alleles output by GATK4. - GATK: enable specification of spark specific parameters with `gatk-spark` resources. - RNA-seq single-cell/DGE: added `demultiplexed` option. If set to True, treat the data as if it has already been demultiplexed into cells/wells. - Multiple orders of magnitude faster templating with thousands of input files. ## 1.1.3 (29 January 2019) - CNV: support background inputs for CNVkit, GATK4 CNV and seq2c. Allows Loading bcbio/bam/__init__.py +3 −3 Original line number Diff line number Diff line Loading @@ -3,7 +3,6 @@ from __future__ import print_function import collections import os import itertools import signal import subprocess import numpy Loading @@ -11,6 +10,7 @@ import numpy import pybedtools import pysam import toolz as tz from six.moves import zip_longest from bcbio import broad, utils from bcbio.bam import ref Loading Loading @@ -48,7 +48,7 @@ def is_paired(bam_file): """ bam_file = objectstore.cl_input(bam_file) cmd = ("set -o pipefail; " "samtools view -h {bam_file} | head -50000 | " "samtools view -h {bam_file} | head -300000 | " "samtools view -S -f 1 /dev/stdin | head -1 | wc -l") p = subprocess.Popen(cmd.format(**locals()), shell=True, executable=do.find_bash(), Loading Loading @@ -291,7 +291,7 @@ def _check_bam_contigs(in_bam, ref_file, config): extra_rcs = [x for x in ref_contigs if x not in bam_contigs] problems = [] warnings = [] for bc, rc in itertools.izip_longest([x for x in bam_contigs if (x not in extra_bcs and for bc, rc in zip_longest([x for x in bam_contigs if (x not in extra_bcs and x not in allowed_outoforder)], [x for x in ref_contigs if (x not in extra_rcs and x not in allowed_outoforder)]): Loading bcbio/bam/callable.py +6 −2 Original line number Diff line number Diff line Loading @@ -198,8 +198,12 @@ def block_regions(callable_bed, in_bam, ref_file, data): if len(ref_regions.subtract(nblock_regions, nonamecheck=True)) > 0: ref_regions.subtract(tx_nblock_bed, nonamecheck=True).merge(d=min_n_size).saveas(tx_callblock_bed) else: raise ValueError("No callable regions found from BAM file. Alignment regions might " "not overlap with regions found in your `variant_regions` BED: %s" % in_bam) raise ValueError("No callable regions found in %s from BAM file %s. Some causes:\n " " - Alignment regions do not overlap with regions found " "in your `variant_regions` BED: %s\n" " - There are no aligned reads in your BAM file that pass sanity checks " " (mapping score > 1, non-duplicates, both ends of paired reads mapped)" % (dd.get_sample_name(data), in_bam, dd.get_variant_regions(data))) return callblock_bed, nblock_bed def _write_bed_regions(data, final_regions, out_file, out_file_ref): Loading bcbio/bam/fastq.py +23 −0 Original line number Diff line number Diff line Loading @@ -7,6 +7,8 @@ from itertools import product import os import random import sys import toolz as tz from collections import defaultdict from Bio import SeqIO from bcbio.distributed import objectstore Loading Loading @@ -115,6 +117,8 @@ def combine_pairs(input_files, force_single=False, full_name=False, separators=N Adjusted to allow different input paths or extensions for matching files. """ PAIR_FILE_IDENTIFIERS = set(["1", "2", "3", "4"]) if len(input_files) > 1000: return fast_combine_pairs(input_files, force_single, full_name, separators) pairs = [] used = set([]) Loading Loading @@ -180,6 +184,25 @@ def combine_pairs(input_files, force_single=False, full_name=False, separators=N used.add(in_file) return pairs def fast_combine_pairs(files, force_single, full_name, separators): """ assume files that need to be paired are within 10 entries of each other, once the list is sorted """ files = sort_filenames(files) chunks = tz.sliding_window(10, files) pairs = [combine_pairs(chunk, force_single, full_name, separators) for chunk in chunks] pairs = [y for x in pairs for y in x] longest = defaultdict(list) # for each file, save the longest pair it is in for pair in pairs: for file in pair: if len(longest[file]) < len(pair): longest[file] = pair # keep only unique pairs longest = {tuple(sort_filenames(x)) for x in longest.values()} # ensure filenames are R1 followed by R2 return [sort_filenames(list(x)) for x in longest] def dif(a, b): """ copy from http://stackoverflow.com/a/8545526 """ return [i for i in range(len(a)) if a[i] != b[i]] Loading Loading
.gitignore +2 −0 Original line number Diff line number Diff line *.pyc *.idx *.swo *.swp bcbio/pipeline/version.py bcbio_nextgen.egg-info/ build/ Loading
HISTORY.md +21 −0 Original line number Diff line number Diff line ## 1.1.4 (3 April 2019) - Move to Python 3.6. A python2 environment in the install runs non python3 compatible programs. The codebase is still compatible with python 2.7 but will only get run and tested on python 3 for future releases. - RNA-seq: fix for race condition when creating the pizzly cache - RNA-seq: Add Salmon to multiqc report. - RNA-seq single-cell/DGE: Properly strip transcript versions from GENCODE GTFs. - RNA-seq: Faster and more flexible rRNA biotype lookup. - Move to R3.5.1, including updates to all CRAN and Bioconductor packages. - tumor-only germline prioritization: provide more useful germline filtering based on prioritization INFO tag (EPR) rather than filter field. - Install: do not require fabric for tool and data installs, making full codebase compatible with python 3. - variant: Filter out variants with missing ALT alleles output by GATK4. - GATK: enable specification of spark specific parameters with `gatk-spark` resources. - RNA-seq single-cell/DGE: added `demultiplexed` option. If set to True, treat the data as if it has already been demultiplexed into cells/wells. - Multiple orders of magnitude faster templating with thousands of input files. ## 1.1.3 (29 January 2019) - CNV: support background inputs for CNVkit, GATK4 CNV and seq2c. Allows Loading
bcbio/bam/__init__.py +3 −3 Original line number Diff line number Diff line Loading @@ -3,7 +3,6 @@ from __future__ import print_function import collections import os import itertools import signal import subprocess import numpy Loading @@ -11,6 +10,7 @@ import numpy import pybedtools import pysam import toolz as tz from six.moves import zip_longest from bcbio import broad, utils from bcbio.bam import ref Loading Loading @@ -48,7 +48,7 @@ def is_paired(bam_file): """ bam_file = objectstore.cl_input(bam_file) cmd = ("set -o pipefail; " "samtools view -h {bam_file} | head -50000 | " "samtools view -h {bam_file} | head -300000 | " "samtools view -S -f 1 /dev/stdin | head -1 | wc -l") p = subprocess.Popen(cmd.format(**locals()), shell=True, executable=do.find_bash(), Loading Loading @@ -291,7 +291,7 @@ def _check_bam_contigs(in_bam, ref_file, config): extra_rcs = [x for x in ref_contigs if x not in bam_contigs] problems = [] warnings = [] for bc, rc in itertools.izip_longest([x for x in bam_contigs if (x not in extra_bcs and for bc, rc in zip_longest([x for x in bam_contigs if (x not in extra_bcs and x not in allowed_outoforder)], [x for x in ref_contigs if (x not in extra_rcs and x not in allowed_outoforder)]): Loading
bcbio/bam/callable.py +6 −2 Original line number Diff line number Diff line Loading @@ -198,8 +198,12 @@ def block_regions(callable_bed, in_bam, ref_file, data): if len(ref_regions.subtract(nblock_regions, nonamecheck=True)) > 0: ref_regions.subtract(tx_nblock_bed, nonamecheck=True).merge(d=min_n_size).saveas(tx_callblock_bed) else: raise ValueError("No callable regions found from BAM file. Alignment regions might " "not overlap with regions found in your `variant_regions` BED: %s" % in_bam) raise ValueError("No callable regions found in %s from BAM file %s. Some causes:\n " " - Alignment regions do not overlap with regions found " "in your `variant_regions` BED: %s\n" " - There are no aligned reads in your BAM file that pass sanity checks " " (mapping score > 1, non-duplicates, both ends of paired reads mapped)" % (dd.get_sample_name(data), in_bam, dd.get_variant_regions(data))) return callblock_bed, nblock_bed def _write_bed_regions(data, final_regions, out_file, out_file_ref): Loading
bcbio/bam/fastq.py +23 −0 Original line number Diff line number Diff line Loading @@ -7,6 +7,8 @@ from itertools import product import os import random import sys import toolz as tz from collections import defaultdict from Bio import SeqIO from bcbio.distributed import objectstore Loading Loading @@ -115,6 +117,8 @@ def combine_pairs(input_files, force_single=False, full_name=False, separators=N Adjusted to allow different input paths or extensions for matching files. """ PAIR_FILE_IDENTIFIERS = set(["1", "2", "3", "4"]) if len(input_files) > 1000: return fast_combine_pairs(input_files, force_single, full_name, separators) pairs = [] used = set([]) Loading Loading @@ -180,6 +184,25 @@ def combine_pairs(input_files, force_single=False, full_name=False, separators=N used.add(in_file) return pairs def fast_combine_pairs(files, force_single, full_name, separators): """ assume files that need to be paired are within 10 entries of each other, once the list is sorted """ files = sort_filenames(files) chunks = tz.sliding_window(10, files) pairs = [combine_pairs(chunk, force_single, full_name, separators) for chunk in chunks] pairs = [y for x in pairs for y in x] longest = defaultdict(list) # for each file, save the longest pair it is in for pair in pairs: for file in pair: if len(longest[file]) < len(pair): longest[file] = pair # keep only unique pairs longest = {tuple(sort_filenames(x)) for x in longest.values()} # ensure filenames are R1 followed by R2 return [sort_filenames(list(x)) for x in longest] def dif(a, b): """ copy from http://stackoverflow.com/a/8545526 """ return [i for i in range(len(a)) if a[i] != b[i]] Loading