Skip to content
Commits on Source (8)
BasedOnStyle: Google
BreakBeforeBraces: Mozilla
AllowShortLoopsOnASingleLine: false
AccessModifierOffset: -4
BreakConstructorInitializersBeforeComma: true
ColumnLimit: 100
IndentWidth: 4
PointerAlignment: Left
TabWidth: 4
ReflowComments: false # protect ASCII art in comments
KeepEmptyLinesAtTheStartOfBlocks: true
......@@ -3,6 +3,7 @@ defines.mk
*.o
*.err
blasr
build/
/utils/bam2bax/build/
/utils/bam2bax/bin/
/utils/bam2bax/tests/bin/
......@@ -11,3 +12,8 @@ blasr
/utils/bax2bam/bin/
/utils/bax2bam/tests/bin/
/utils/bax2bam/tests/src/TestData.h
# Meson WrapDB stuff
subprojects/packagecache/
subprojects/blasr_libcpp*
subprojects/pbbam*
[submodule "libcpp"]
path = libcpp
url = git://github.com/PacificBiosciences/blasr_libcpp.git
This diff is collapsed.
// Copyright (c) 2016, Pacific Biosciences of California, Inc.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted (subject to the limitations in the
// disclaimer below) provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following
// disclaimer in the documentation and/or other materials provided
// with the distribution.
//
// * Neither the name of Pacific Biosciences nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
// BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
// USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
// SUCH DAMAGE.
// Author: Armin Töpfer
#pragma once
#include <string>
namespace PacBio {
namespace {
static constexpr const char BlasrGitSha1Int[] = "@BLASR_GIT_SHA1@";
}
inline std::string BlasrGitSha1()
{
if (sizeof(BlasrGitSha1Int) == 1) {
// corresponds to a release
return std::string();
} else {
// we have a git SHA1
return std::string("-") + BlasrGitSha1Int;
}
}
} // ::PacBio
// Copyright (c) 2016, Pacific Biosciences of California, Inc.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted (subject to the limitations in the
// disclaimer below) provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following
// disclaimer in the documentation and/or other materials provided
// with the distribution.
//
// * Neither the name of Pacific Biosciences nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
// GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
// BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
// USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
// OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
// SUCH DAMAGE.
// Author: Armin Töpfer
#pragma once
#include <algorithm>
#include <string>
#include "BlasrGitHash.h"
namespace PacBio {
namespace {
static constexpr const char BlasrVersionInt[] = "@BLASR_VERSION@";
static constexpr const char BlasrMajorVersionInt[] = "@BLASR_VERSION_MAJOR@.@BLASR_VERSION_MINOR@";
static constexpr const char BlasrChangelogInt[] = "${BLASR_CHANGELOG}";
static constexpr const size_t BlasrVersionMajorInt = @BLASR_VERSION_MAJOR@;
static constexpr const size_t BlasrVersionMinorInt = @BLASR_VERSION_MINOR@;
static constexpr const size_t BlasrVersionPatchInt = @BLASR_VERSION_PATCH@;
}
inline std::string BlasrVersion() { return BlasrVersionInt; }
inline std::string BlasrMajorVersion() { return BlasrMajorVersionInt; }
} // ::PacBio
Copyright (c) 2011-2015, Pacific Biosciences of California, Inc.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Pacific Biosciences nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE
GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY PACIFIC
BIOSCIENCES AND ITS CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL PACIFIC BIOSCIENCES OR ITS CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
## Installation
### See a step by step Blasr installation example on Blasr wiki page
https://github.com/PacificBiosciences/blasr/wiki/Step-by-step-blasr-installation-example
### Download source code
* To pull this project from git hub to your local system:
git clone git://github.com/PacificBiosciences/blasr.git blasr
* To sync your code with the latest git code base:
git pull --rebase origin master && git submodule update --init
* To update the submodule:
make update-submodule
### Requirements
* To configure:
./configure.py --shared --sub --no-pbbam
* or with HDF5 directories (and note that `HDF5_LIB` is a *directory* here):
./configure.py --shared --sub --no-pbbam HDF5_INCLUDE=... HDF5_LIB=...
To build BLASR, you must have hdf 1.8.12 or above installed and
configured with c++ support (you should have the library
libhdf5_cpp.a). If you are intalling the entire PacBio secondary
analysis software suite, appropriate hdf libraries are already
distributed and no configuration is necessary. Otherwise, it is
necessary to point two environment variables:
+ **HDF5_INCLUDE**, which points to directory of the HDF5 headers
(e.g., hdf5.h)
+ **HDF5_LIB**, which points to the HDF5 library directory (e.g., hdf5*.a,
and hdf5*.so)
You may pass arguments to `configure.py` as above, or you may export them from command line:
export HDF5_INC=path_to_your_hdf5_include && export HDF5_LIB=path_to_your_hdf5_lib
* To configure submodule:
make configure-submodule
### Build
* To make the 'libcpp' libraries:
make build-submodule
* To make 'blasr' only:
make blasr
* To compile all tools, including blasr, pls2fasta, loadPulses, sawriter:
make
* Frequently used executables will be under utils.
* To test (with **cram** installed):
#make cramtests
make cramfast
## Currently:
## Ran 22 tests, 0 skipped, 4 failed.
* To clean all compiled tools and lib:
make cleanall
* To clean compiled tools without cleaning lib:
make clean
make blasr
./blasr
### CXXFLAGS
* For optimized builds:
./configure.py CXXFLAGS=-O3 ...
* For debug builds:
./configure.py CXXFLAGS=-g ...
## Other issues
### Static binaries
If you want static binaries, drop `--shared` when you run configure.py. In that case, you
might need to pass `-lsz` to make, if you built HDF5 with szlib support (`--with-szlib`).
./configure.py --with-szlib ...
See [our issues](https://github.com/PacificBiosciences/blasr/issues/113#issuecomment-143981496).
If you have macosx (Darwin), then you almost certainly want non-static binaries (--shared).
### blasr_libcpp
If you have built and installed blasr_libcpp elsewhere, then drop `--sub` and do not run `make build-submodule`.
See Blasr [README.INSTALL.md](README.INSTALL.md) and [README.MANUAL.md](README.MANUAL.md).
<p align="center">
<img src="doc/img/blasr.png" alt="blasr logo" width="200px"/>
</p>
<h1 align="center">BLASR</h1>
<p align="center">The PacBio® long read aligner</p>
***
## Availability
The latest release binary can be installed via [bioconda](https://bioconda.github.io/):
conda install -c bioconda blasr
Official support is only provided for official and stable
[SMRT Analysis builds](http://www.pacb.com/products-and-services/analytical-software/)
provided by PacBio and not for source builds.
Unofficial support for binary releases is provided via github issues,
not via mail to developers.
**No** support for source builds; issues will be closed without comment.
## Documentation
- [Wiki: all you need to know](https://github.com/PacificBiosciences/blasr/wiki)
- [Running BLASR](doc/EXAMPLES.md)
- [Developer environment](doc/DEVELOPER.md)
DISCLAIMER
----------
THIS WEBSITE AND CONTENT AND ALL SITE-RELATED SERVICES, INCLUDING ANY DATA, ARE PROVIDED "AS IS," WITH ALL FAULTS, WITH NO REPRESENTATIONS OR WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF MERCHANTABILITY, SATISFACTORY QUALITY, NON-INFRINGEMENT OR FITNESS FOR A PARTICULAR PURPOSE. YOU ASSUME TOTAL RESPONSIBILITY AND RISK FOR YOUR USE OF THIS SITE, ALL SITE-RELATED SERVICES, AND ANY THIRD PARTY WEBSITES OR APPLICATIONS. NO ORAL OR WRITTEN INFORMATION OR ADVICE SHALL CREATE A WARRANTY OF ANY KIND. ANY REFERENCES TO SPECIFIC PRODUCTS OR SERVICES ON THE WEBSITES DO NOT CONSTITUTE OR IMPLY A RECOMMENDATION OR ENDORSEMENT BY PACIFIC BIOSCIENCES.
#!/usr/bin/env bash
set -vex
################
# DEPENDENCIES #
################
## Load modules
type module >& /dev/null || . /mnt/software/Modules/current/init/bash
module purge
module load gcc
module load ccache
module load meson
module load ninja
module load boost
module load hdf5-tools
module load cram
case "${bamboo_planRepository_branchName}" in
master)
module load libblasr/master
module load pbbam/master
;;
*)
module load libblasr/develop
module load pbbam/develop
;;
esac
BOOST_ROOT="${BOOST_ROOT%/include}"
# unset these variables to have meson discover all
# boost-dependent variables from BOOST_ROOT alone
unset BOOST_INCLUDEDIR
unset BOOST_LIBRARYDIR
export CC="ccache gcc"
export CXX="ccache g++"
export CCACHE_BASEDIR="${PWD}"
if [[ $USER == bamboo ]]; then
export CCACHE_DIR=/mnt/secondary/Share/tmp/bamboo.${bamboo_shortPlanKey}.ccachedir
export CCACHE_TEMPDIR=/scratch/bamboo.ccache_tempdir
fi
case "${bamboo_planRepository_branchName}" in
develop|master)
export PREFIX_ARG="/mnt/software/b/blasr/${bamboo_planRepository_branchName}"
export BUILD_NUMBER="${bamboo_globalBuildNumber:-0}"
;;
*)
export BUILD_NUMBER="0"
;;
esac
# call the main build+test scripts
export CURRENT_BUILD_DIR="build"
export ENABLED_TESTS="true"
# TODO(dseifert)
# HDF5 doesn't have pkg-config files yet
export CPPFLAGS="${HDF5_CFLAGS}"
export LDFLAGS="-static-libstdc++ -static-libgcc ${HDF5_LIBS}"
bash scripts/ci/build.sh
bash scripts/ci/test.sh
if [[ -z ${PREFIX_ARG+x} ]]; then
echo "Not installing anything (branch: ${bamboo_planRepository_branchName}), exiting."
exit 0
fi
bash scripts/ci/install.sh
import com.atlassian.bamboo.specs.api.BambooSpec;
import com.atlassian.bamboo.specs.api.builders.AtlassianModule;
import com.atlassian.bamboo.specs.api.builders.BambooKey;
import com.atlassian.bamboo.specs.api.builders.BambooOid;
import com.atlassian.bamboo.specs.api.builders.plan.Job;
import com.atlassian.bamboo.specs.api.builders.plan.Plan;
import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
import com.atlassian.bamboo.specs.api.builders.plan.Stage;
import com.atlassian.bamboo.specs.api.builders.plan.artifact.Artifact;
import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
import com.atlassian.bamboo.specs.api.builders.project.Project;
import com.atlassian.bamboo.specs.api.builders.requirement.Requirement;
import com.atlassian.bamboo.specs.api.builders.task.AnyTask;
import com.atlassian.bamboo.specs.api.model.BambooKeyProperties;
import com.atlassian.bamboo.specs.api.model.BambooOidProperties;
import com.atlassian.bamboo.specs.api.model.plan.JobProperties;
import com.atlassian.bamboo.specs.api.model.plan.PlanIdentifierProperties;
import com.atlassian.bamboo.specs.api.model.plan.PlanProperties;
import com.atlassian.bamboo.specs.api.model.plan.StageProperties;
import com.atlassian.bamboo.specs.api.model.plan.artifact.ArtifactProperties;
import com.atlassian.bamboo.specs.api.model.plan.branches.PlanBranchManagementProperties;
import com.atlassian.bamboo.specs.api.model.plan.requirement.RequirementProperties;
import com.atlassian.bamboo.specs.api.model.project.ProjectProperties;
import com.atlassian.bamboo.specs.api.model.task.AnyTaskProperties;
import com.atlassian.bamboo.specs.builders.task.ArtifactDownloaderTask;
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
import com.atlassian.bamboo.specs.builders.task.DownloadItem;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.builders.task.TestParserTask;
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
import com.atlassian.bamboo.specs.builders.trigger.RemoteTrigger;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.model.task.ArtifactDownloaderTaskProperties;
import com.atlassian.bamboo.specs.model.task.DownloadItemProperties;
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties;
import com.atlassian.bamboo.specs.model.task.TestParserTaskProperties;
import com.atlassian.bamboo.specs.model.task.VcsCheckoutTaskProperties;
import com.atlassian.bamboo.specs.model.trigger.BitbucketServerTriggerProperties;
import com.atlassian.bamboo.specs.model.trigger.RemoteTriggerProperties;
import com.atlassian.bamboo.specs.model.trigger.ScheduledTriggerProperties;
import com.atlassian.bamboo.specs.util.BambooServer;
import com.atlassian.bamboo.specs.util.MapBuilder;
@BambooSpec
public class PlanSpec {
public static void main(String... argv) {
//By default credentials are read from the '.credentials' file.
BambooServer bambooServer = new BambooServer("http://localhost:8085");
Plan rootObject = new Plan(new Project()
.oid(new BambooOid("1jx1kzc2o5xc1"))
.key(new BambooKey("PLAYG")),
"blasr-move bamboo inline build scripts to bitbucket",
new BambooKey("BBIBSTB"))
.oid(new BambooOid("1jwrvrqpgcf88"))
.enabled(false)
.stages(new Stage("sonarqube")
.jobs(new Job("SonarQube",
new BambooKey("SON"))
.tasks(new VcsCheckoutTask()
.description("Checkout Default Repository")
.checkoutItems(new CheckoutItem().defaultRepository()),
new AnyTask(new AtlassianModule("ch.mibex.bamboo.sonar4bamboo:sonar4bamboo.sonarscannertask"))
.configuration(new MapBuilder()
.put("incrementalFileForInclusionList", "")
.put("chosenSonarConfigId", "4")
.put("useGradleWrapper", "")
.put("useNewGradleSonarQubePlugin", "")
.put("sonarJavaSource", "")
.put("sonarProjectName", "SAT-blasr")
.put("buildJdk", "JDK 1.8.0_101")
.put("gradleWrapperLocation", "")
.put("sonarLanguage", "")
.put("sonarSources", "iblasr, ./Blasr.cpp")
.put("useGlobalSonarServerConfig", "true")
.put("incrementalMode", "")
.put("failBuildForBrokenQualityGates", "")
.put("sonarTests", "")
.put("failBuildForSonarErrors", "")
.put("sonarProjectVersion", "5.3.0.${bamboo.buildNumber}")
.put("sonarBranch", "")
.put("executable", "SonarScannerHome")
.put("illegalBranchCharsReplacement", "_")
.put("failBuildForTaskErrors", "true")
.put("incrementalModeNotPossible", "incrementalModeRunFullAnalysis")
.put("sonarJavaTarget", "")
.put("environmentVariables", "")
.put("incrementalModeGitBranchPattern", "")
.put("legacyBranching", "")
.put("replaceSpecialBranchChars", "")
.put("additionalProperties", "-Dsonar.cfamily.build-wrapper-output.bypass=true")
.put("autoBranch", "true")
.put("sonarProjectKey", "SAT-blasr")
.put("incrementalModeBambooUser", "")
.put("overrideSonarBuildConfig", "true")
.put("workingSubDirectory", "")
.build()))
.requirements(new Requirement("system.os")
.matchValue("linux")
.matchType(Requirement.MatchType.EQUALS))),
new Stage("Default Stage")
.jobs(new Job("Building",
new BambooKey("JOB1"))
.artifacts(new Artifact()
.name("prebuilt_blasr")
.copyPattern("blasr-*.tgz")
.location("tarballs"))
.tasks(new VcsCheckoutTask()
.description("Checkout Default Repository")
.checkoutItems(new CheckoutItem().defaultRepository())
.cleanCheckout(true),
new ArtifactDownloaderTask()
.description("pbbam artifact")
.enabled(false)
.sourcePlan(new PlanIdentifier()
.key(new BambooKey("PBBAM"))
.projectKey(new BambooKey("SAT")))
.artifacts(new DownloadItem()
.allArtifacts(true)),
new ArtifactDownloaderTask()
.description("blasr_libcpp artifact")
.enabled(false)
.sourcePlan(new PlanIdentifier()
.key(new BambooKey("BLC"))
.projectKey(new BambooKey("SAT")))
.artifacts(new DownloadItem()
.allArtifacts(true)),
new ScriptTask()
.description("build, test, and publish")
.location(ScriptTaskProperties.Location.FILE)
.fileFromPath("bamboo_build.sh"),
new ScriptTask()
.description("test_fast")
.enabled(false)
.inlineBody("#!/bin/bash -xe\nexec bash repos/blasr/bamboo_test.sh"),
new TestParserTask(TestParserTaskProperties.TestType.JUNIT)
.description("run_cram_unit")
.resultDirectories("build/*.xml"))
.requirements(new Requirement("system.dist")
.matchValue("redhat")
.matchType(Requirement.MatchType.EQUALS))))
.linkedRepositories("blasr")
.triggers(new BitbucketServerTrigger()
.name("Bitbucket Server repository triggered"),
new RemoteTrigger()
.name("Remote trigger"),
new ScheduledTrigger()
.name("Scheduled")
.description("nightly")
.enabled(false))
.planBranchManagement(new PlanBranchManagement()
.createForPullRequest()
.delete(new BranchCleanup()
.whenRemovedFromRepositoryAfterDays(7)
.whenInactiveInRepositoryAfterDays(30))
.notificationForCommitters());
bambooServer.publish(rootObject);
}
}
#!/usr/bin/env python
"""Configure the build.
- Create defines.mk
"""
import commands
import contextlib
import optparse
import os
import sys
import warnings
#DEFAULTCXXFLAG := -O3
#DEBUGCXXFLAG := -g -ggdb -fno-inline
#PROFILECXXFLAG := -Os -pg
#GCXXFLAG := -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fno-omit-frame-pointer
ROOT = os.path.abspath(os.path.dirname(__file__))
def log(msg):
sys.stderr.write(msg)
sys.stderr.write('\n')
def shell(cmd):
log('`%s`'%cmd)
status, output = commands.getstatusoutput(cmd)
if status:
raise Exception('%d <- %r' %(status, cmd))
log(output)
return output
def system(cmd):
log(cmd)
status = os.system(cmd)
if status:
raise Exception('%d <- %r' %(status, cmd))
return
def mkdirs(path):
if not os.path.isdir(path):
os.makedirs(path)
@contextlib.contextmanager
def cd(nwd):
cwd = os.getcwd()
log('cd %r -> %r' %(cwd, nwd))
os.chdir(nwd)
yield
os.chdir(cwd)
log('cd %r <- %r' %(cwd, nwd))
def update_content(fn, content):
current_content = open(fn).read() if os.path.exists(fn) else None
if content != current_content:
log('writing to %r:' %fn)
log('"""\n' + content + '"""\n----')
open(fn, 'w').write(content)
def get_OS_STRING():
G_BUILDOS_CMD = """bash -c 'set -e; set -o pipefail; id=$(lsb_release -si | tr "[:upper:]" "[:lower:]"); rel=$(lsb_release -sr); case $id in ubuntu) printf "$id-%04d\n" ${rel/./};; centos) echo "$id-${rel%%.*}";; *) echo "$id-$rel";; esac' 2>/dev/null"""
return shell(G_BUILDOS_CMD)
def get_PREBUILT():
cmd = 'cd ../../../../prebuilt.out 2>/dev/null && pwd || echo -n notfound'
return shell(cmd)
def ifenvf(env, key, func):
if key in env:
return env[key]
else:
return func()
def setifenvf(envout, envin, key, func):
envout[key] = ifenvf(envin, key, func)
def setifenv(envout, envin, key, val):
envout[key] = envin.get(key, val)
def setenv(envout, key, val):
envout[key] = val
def update_env_if(envout, envin, keys):
for key in keys:
if key in envin:
envout[key] = envin[key]
def compose_defs_env(env):
# We disallow env overrides for some things with defaults from GNU make.
nons = ['CXX', 'CC', 'AR'] # 'SHELL'?
ovr = ['%-20s ?= %s' %(k, v) for k,v in env.items() if k not in nons]
nonovr = ['%-20s := %s' %(k, v) for k,v in env.items() if k in nons]
return '\n'.join(ovr + nonovr + [''])
def compose_defines_pacbio(envin):
"""
This is used by mobs via buildcntl.sh.
"""
env = dict()
setenv(env, 'SHELL', 'bash')
#setifenvf(env, envin, 'OS_STRING', get_OS_STRING)
#setifenvf(env, envin, 'PREBUILT', get_PREBUILT)
nondefaults = set([
'CXX',
'CXXFLAGS',
'NOPBBAM',
'LIBPBDATA_INC', 'LIBPBDATA_LIB', 'LIBPBDATA_LIBFLAGS',
'LIBPBIHDF_INC', 'LIBPBIHDF_LIB', 'LIBPBIHDF_LIBFLAGS',
'LIBBLASR_INC', 'LIBBLASR_LIB', 'LIBBLASR_LIBFLAGS',
'HDF5_INC', 'HDF5_LIB', 'HDF5_LIBFLAGS',
'PBBAM_INC', 'PBBAM_LIB', 'PBBAM_LIBFLAGS',
'HTSLIB_INC', 'HTSLIB_LIB', 'HTSLIB_LIBFLAGS',
'BOOST_INC',
'GCC_LIB',
'ZLIB_LIB', 'ZLIB_LIBFLAGS',
'SZLIB_LIB', 'SZLIB_LIBFLAGS',
'PTHREAD_LIBFLAGS',
'DL_LIBFLAGS',
'RT_LIBFLAGS',
])
update_env_if(env, envin, nondefaults)
return compose_defs_env(env)
def configure_pacbio(envin, shared, build_dir):
content1 = compose_defines_pacbio(envin)
if not shared:
content1 += 'LDFLAGS+=-static\n'
update_content(os.path.join(build_dir, 'defines.mk'), content1)
def set_defs_submodule_defaults(env, nopbbam):
subdir = os.path.join(ROOT, 'libcpp')
defaults = {
'LIBPBDATA_INC': os.path.join(subdir, 'pbdata'),
'LIBBLASR_INC': os.path.join(subdir, 'alignment'),
#'LIBPBIHDF_INC': '' if nopbbam else os.path.join(subdir, 'hdf'),
'LIBPBDATA_LIB': os.path.join(subdir, 'pbdata'),
'LIBBLASR_LIB': os.path.join(subdir, 'alignment'),
#'LIBPBIHDF_LIB': '' if nopbbam else os.path.join(subdir, 'hdf'),
}
for k in defaults:
if k not in env:
env[k] = defaults[k]
def update_defaults_for_os(env):
OS = shell('uname')
if 'Darwin' in OS:
#-lsz (for static builds?)
env['RT_LIBFLAGS'] = ''
def set_defs_defaults(env, nopbbam, with_szlib):
defaults = {
'LIBBLASR_INC': os.path.join(ROOT, 'libcpp', 'alignment'),
'LIBPBDATA_INC': os.path.join(ROOT, 'libcpp', 'pbdata'),
'LIBPBIHDF_INC': os.path.join(ROOT, 'libcpp', 'hdf'),
'LIBBLASR_LIB': os.path.join(ROOT, 'libcpp', 'alignment'),
'LIBPBDATA_LIB': os.path.join(ROOT, 'libcpp', 'pbdata'),
'LIBPBIHDF_LIB': os.path.join(ROOT, 'libcpp', 'hdf'),
'LIBBLASR_LIBFLAGS': '-lblasr',
'LIBPBDATA_LIBFLAGS': '-lpbdata',
'LIBPBIHDF_LIBFLAGS': '-lpbihdf',
'HDF5_LIBFLAGS': '-lhdf5_cpp -lhdf5',
'RT_LIBFLAGS': '-lrt',
'ZLIB_LIBFLAGS': '-lz',
'PTHREAD_LIBFLAGS': '-lpthread',
'DL_LIBFLAGS': '-ldl', # neeeded by HDF5 always
'SHELL': 'bash -xe',
}
try:
update_defaults_for_os(defaults)
except Exception as e:
warnings.warn(e)
#setifenvf(defaults, env, 'OS_STRING', get_OS_STRING)
#setifenvf(defaults, env, 'PREBUILT', get_PREBUILT)
pbbam_defaults = {
'PBBAM_LIBFLAGS': '-lpbbam',
'HTSLIB_LIBFLAGS': '-lhts',
'ZLIB_LIBFLAGS': '-lz',
#'PTHREAD_LIBFLAGS': '-lpthread',
#'DL_LIBFLAGS': '-ldl', # neeeded by HDF5 always
}
if not nopbbam:
defaults.update(pbbam_defaults)
else:
defaults['NOPBBAM'] = 1
szlib_defaults = {
'SZLIB_LIBFLAGS': '-lsz',
#'ZLIB_LIBFLAGS': '-lz', # probably needed, but provided elsewhere
}
if with_szlib:
defaults.update(szlib_defaults)
for k in defaults:
if k not in env:
env[k] = defaults[k]
def get_make_style_env(envin, args):
envout = dict()
for arg in args:
if '=' in arg:
k, v = arg.split('=')
envout[k] = v
envout.update(envin)
return envout
def parse_args(args):
parser = optparse.OptionParser()
parser.add_option('--no-pbbam', action='store_true',
help='Avoid compiling anything which would need pbbam.')
parser.add_option('--with-szlib', action='store_true',
help='If HDF5 was built with --with-szlib, then -lsz is needed for static binaries.')
parser.add_option('--submodules', action='store_true',
help='Set variables to use our git-submodules, which must be pulled and built first. (Implies --no-pbbam.)')
parser.add_option('--shared', action='store_true',
help='Build for dynamic linking. (Non-static binaries.)')
parser.add_option('--build-dir',
help='Can be different from source directory, but only when *not* also building submodule.')
return parser.parse_args(list(args))
def symlink_makefile(build_dir_root, src_dir_root, makefilename, relpath):
src_dir = os.path.join(src_dir_root, relpath)
build_dir = os.path.join(build_dir_root, relpath)
src_name = os.path.join(src_dir, 'makefile')
dst_name = os.path.join(build_dir, 'makefile')
if os.path.lexists(dst_name):
os.unlink(dst_name)
print('%r <- %r' %(src_name, dst_name))
mkdirs(build_dir)
os.symlink(src_name, dst_name)
def symlink_makefiles(build_dir):
symlink_makefile(build_dir, ROOT, 'makefile', '.')
symlink_makefile(build_dir, ROOT, 'makefile', 'utils')
symlink_makefile(build_dir, ROOT, 'makefile', 'extrautils')
def main(prog, *args):
"""We are still deciding what env-vars to use, if any.
"""
# Set up an alias, until everything uses one consistently.
conf, makevars = parse_args(args)
if conf.build_dir is not None:
symlink_makefiles(conf.build_dir)
else:
conf.build_dir = '.'
conf.build_dir = os.path.abspath(conf.build_dir)
envin = get_make_style_env(os.environ, makevars)
if 'HDF5_INCLUDE' in envin and 'HDF5_INC' not in envin:
envin['HDF5_INC'] = envin['HDF5_INCLUDE']
if conf.submodules:
set_defs_submodule_defaults(envin, conf.no_pbbam)
conf.no_pbbam = True
set_defs_defaults(envin, conf.no_pbbam, conf.with_szlib)
configure_pacbio(envin, conf.shared, conf.build_dir)
if __name__=="__main__":
main(*sys.argv)
FAST_CTESTS := \
ctest/ecoli.t \
ctest/fastMaxInterval.t \
ctest/aggressiveIntervalCut.t \
ctest/multipart.t \
ctest/affineAlign.t ctest/bamOut.t ctest/ccsH5.t ctest/filtercriteria.t ctest/m0-5.t \
ctest/fofn.t \
ctest/alignScore.t ctest/hitpolicy.t ctest/noSplitSubreads.t \
ctest/bamIn.t ctest/open_fail.t ctest/verbose.t ctest/deterministic.t
MILD_CTESTS := \
ctest/concordant.t ctest/bug25766.t ctest/holeNumbers.t
SLOW_CTESTS := ctest/bug25328.t
# XXX: following tests sidelined, needs bam input after --sam option removed
# MILD: ctest/useccsallBestN1.t
# sidelined because of changes in directories
#
# needed to restore /mnt/data3/vol53/2450530/0014
# SLOW ctest/useccsallLargeGenome.t
#BLASR_PATH=/mnt/secondary/builds/full/3.0.0/prod/current-build_smrtanalysis/private/otherbins/internalall/bin/
#export BLASR_PATH
cramfast:
cram -v --shell=/bin/bash ${FAST_CTESTS}
crammild:
cram -v --shell=/bin/bash ${MILD_CTESTS}
cramslow:
cram -v --shell=/bin/bash ${SLOW_CTESTS}
cramtests:
cram -v --shell=/bin/bash ${FAST_CTESTS} ${MILD_CTESTS} ${SLOW_CTESTS}
cramqu:
for test in ${FAST_CTESTS}; do \
qsub -pe smp 15 -V -cwd -b y -N cramqu $@cram -v --shell=bin/bash $$test;\
done
clean:
rm -f cramqu.* ctest/*.err
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
Test affineAlign
$ rm -rf $OUTDIR/affineAlign.m0
$ $EXEC $DATDIR/affineAlign.fofn $DATDIR/substr_with_ins.fasta -m 0 --out $OUTDIR/affineAlign.m0 --affineAlign --holeNumbers 493 --insertion 100 --deletion 100
$ $BLASR_EXE $DATDIR/affineAlign.fofn $DATDIR/substr_with_ins.fasta -m 0 --out $OUTDIR/affineAlign.m0 --affineAlign --holeNumbers 493 --insertion 100 --deletion 100
[INFO]* (glob)
[INFO]* (glob)
$ diff $OUTDIR/affineAlign.m0 $STDDIR/affineAlign_2014_06_10.m0
$ rm -rf $OUTDIR/ecoli_affine.m0
$ $EXEC $DATDIR/ecoli_affine.fasta $DATDIR/ecoli_reference.fasta -m 0 --out $OUTDIR/ecoli_affine.m0 --affineAlign --insertion 100 --deletion 100
$ $BLASR_EXE $DATDIR/ecoli_affine.fasta $DATDIR/ecoli_reference.fasta -m 0 --out $OUTDIR/ecoli_affine.m0 --affineAlign --insertion 100 --deletion 100
[INFO]* (glob)
[INFO]* (glob)
$ diff $OUTDIR/ecoli_affine.m0 $STDDIR/ecoli_affine_2014_06_10.m0
......
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
Test --aggressiveIntervalCut.
$ rm -f $TMP1
$ BASFILE=/pbi/dept/secondary/siv/testdata/BlasrTestData/ctest/data/aggressiveIntervalCut/m130812_185809_42141_c100533960310000001823079711101380_s1_p0.bas.h5
$ REFFA=/pbi/dept/secondary/siv/testdata/BlasrTestData/ctest/data/references/Ecoli_BL21_O26/sequence/Ecoli_BL21_O26.fasta
$ $EXEC $BASFILE $REFFA --holeNumbers 1--100 --out $TMP1 --aggressiveIntervalCut
$ $BLASR_EXE $BASFILE $REFFA --holeNumbers 1--100 --out $TMP1 --aggressiveIntervalCut
[INFO] * [blasr] started. (glob)
[INFO] * [blasr] ended. (glob)
$ echo $?
......
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
Test alignment score
$ rm -rf $OUTDIR/testscore.m0
$ $EXEC $DATDIR/read.fasta $DATDIR/ref.fasta --minReadLength 1 -m 0 --out $OUTDIR/testscore.m0
$ $BLASR_EXE $DATDIR/read.fasta $DATDIR/ref.fasta --minReadLength 1 -m 0 --out $OUTDIR/testscore.m0
[INFO]* (glob)
[INFO]* (glob)
$ diff $OUTDIR/testscore.m0 $STDDIR/testscore.m0
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
Test using bam as input, use --concordant
$ $EXEC $DATDIR/test_bam/tiny_bam.fofn $DATDIR/bamConcordantRef.fasta --bam --concordant --refineConcordantAlignments --bestn 1 --out $OUTDIR/bamConcordant.bam
$ $BLASR_EXE $DATDIR/test_bam/tiny_bam.fofn $DATDIR/bamConcordantRef.fasta --bam --concordant --refineConcordantAlignments --bestn 1 --out $OUTDIR/bamConcordant.bam
[INFO]* (glob)
[INFO]* (glob)
Check whether sam out and bam out have identical alignments, not checking qvs
$ $SAMTOOLS view $OUTDIR/bamConcordant.bam |cut -f 4
$ $SAMTOOLS_EXE view $OUTDIR/bamConcordant.bam |cut -f 4
1
1
8??? (glob)
......@@ -25,7 +25,7 @@ Check whether sam out and bam out have identical alignments, not checking qvs
86?? (glob)
86?? (glob)
$ $EXEC /pbi/dept/secondary/siv/testdata/SA3-RS/lambda/2372215/0007_tiny/Analysis_Results/m150404_101626_42267_c100807920800000001823174110291514_s1_p0.1.subreads.bam $DATDIR/lambda_ref.fasta -m 4 --concordant --bestn 1 --holeNumbers 17417 --out $OUTDIR/tmp.m4 -V 2 > $OUTDIR/bamConcordant.log
$ $BLASR_EXE /pbi/dept/secondary/siv/testdata/SA3-RS/lambda/2372215/0007_tiny/Analysis_Results/m150404_101626_42267_c100807920800000001823174110291514_s1_p0.1.subreads.bam $DATDIR/lambda_ref.fasta -m 4 --concordant --bestn 1 --holeNumbers 17417 --out $OUTDIR/tmp.m4 -V 2 > $OUTDIR/bamConcordant.log
[INFO]* (glob)
[INFO]* (glob)
......
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
Test using bam as input
$ $EXEC $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta -m 4 --out $OUTDIR/tiny_bam_in.m4
$ $BLASR_EXE $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta -m 4 --out $OUTDIR/tiny_bam_in.m4
[INFO]* (glob)
[INFO]* (glob)
Check whether blasr produces identical results taking fasta sequences of the bam as input
$ $EXEC $DATDIR/test_bam/tiny_fasta.fofn $DATDIR/lambda_ref.fasta -m 4 --out $OUTDIR/tiny_fasta_in.m4
$ $BLASR_EXE $DATDIR/test_bam/tiny_fasta.fofn $DATDIR/lambda_ref.fasta -m 4 --out $OUTDIR/tiny_fasta_in.m4
[INFO]* (glob)
[INFO]* (glob)
$ diff $OUTDIR/tiny_bam_in.m4 $OUTDIR/tiny_fasta_in.m4
Test bam in, bam out
$ $EXEC $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta --bam --out $OUTDIR/tiny_bam_in.bam --clipping subread
$ $BLASR_EXE $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta --bam --out $OUTDIR/tiny_bam_in.bam --clipping subread
[INFO]* (glob)
[INFO]* (glob)
......
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
Test generating bam output
Input is bam, clipping=soft and subread should produce identical results
$ $EXEC $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta --bam --out $OUTDIR/tiny_bam_in_soft.bam --clipping soft
$ $BLASR_EXE $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta --bam --out $OUTDIR/tiny_bam_in_soft.bam --clipping soft
[INFO]* (glob)
[INFO]* (glob)
$ $EXEC $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta --bam --out $OUTDIR/tiny_bam_in_subread.bam --clipping subread
$ $BLASR_EXE $DATDIR/test_bam/tiny_bam.fofn $DATDIR/lambda_ref.fasta --bam --out $OUTDIR/tiny_bam_in_subread.bam --clipping subread
[INFO]* (glob)
[INFO]* (glob)
$ $SAMTOOLS view $OUTDIR/tiny_bam_in_soft.bam | sed -n '6,$p' > $TMP1.bam_in_soft
$ $SAMTOOLS view $OUTDIR/tiny_bam_in_subread.bam | sed -n '6,$p' > $TMP2.bam_in_subread
$ $SAMTOOLS_EXE view $OUTDIR/tiny_bam_in_soft.bam | sed -n '6,$p' > $TMP1.bam_in_soft
$ $SAMTOOLS_EXE view $OUTDIR/tiny_bam_in_subread.bam | sed -n '6,$p' > $TMP2.bam_in_subread
$ diff $TMP1.bam_in_soft $TMP2.bam_in_subread
Test if bam cigar strings are correct
......
Set up
$ . $TESTDIR/setup.sh
$ mkdir -p $OUTDIR
bug_25328, unrolled resequencing test
$ INFA=$DATDIR/bug_25328_zmw_38131.fasta
$ REF=$DATDIR/All4mers_circular_72x_l50256.fasta
$ OUTFA=$OUTDIR/bug_25328.m4
$ $EXEC $INFA $REF --bestn 1 --nCandidates 1 --forwardOnly --maxMatch 14 -m 4 --out $OUTFA
$ $BLASR_EXE $INFA $REF --bestn 1 --nCandidates 1 --forwardOnly --maxMatch 14 -m 4 --out $OUTFA
[INFO]* (glob)
[INFO]* (glob)
......