...
 
Commits (31)
......@@ -4,6 +4,8 @@
language: erlang
cache:
- apt
services:
- docker
script:
# Test the ultimate one-liner setup for NeuroDebian repository
- bash tools/neurodebian-travis.sh
......@@ -21,6 +23,8 @@ script:
- test -f /etc/apt/sources.list.d/neurodebian.sources.list
# Verify that -dev installs nicely
- sudo apt-get install -y neurodebian-dev
# Test nd_freeze
- bash tools/tests/test_nd_freeze
after_failure:
# Could happen if installation of neurodebian package kills APT repo so print policy
- sudo apt-cache policy
......
......@@ -107,13 +107,13 @@ debian = http://http.debian.net/debian
ubuntu = http://de.archive.ubuntu.com/ubuntu
[task files]
datasets = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/datasets
distributedcomputing = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/distributedcomputing
education = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/education
electrophysiology = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/electrophysiology
mri = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/mri
neuralmodeling = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/neuralmodeling
psychophysics = https://salsa.debian.org/neurodebian-team/neurodebian-blend.git/plain/tasks/psychophysics
datasets = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/datasets
distributedcomputing = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/distributedcomputing
education = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/education
electrophysiology = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/electrophysiology
mri = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/mri
neuralmodeling = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/neuralmodeling
psychophysics = https://salsa.debian.org/neurodebian-team/neurodebian-blend/raw/master/tasks/psychophysics
[release names]
# Map code names to release names and description
......
# Copyright (c) 2018, Yaroslav O. Halchenko. All rights reserved. MIT license
#
# The purpose of the Singularity environment is to provide a relatively full
# suite of tools provided primarily by Debian/NeuroDebian for runnin various
# neuroimaging analyses.
#
# Notes:
# - Due to https://github.com/singularityware/singularity/issues/471
# bootstrapping leads to non-usable/non-removable-without-reboot
# image due to some rogue run away processes.
# This line could help to kill them but should be used with caution
# since could kill other unrelated processes
#
# grep -l loop /proc/*/mountinfo | sed -e 's,/proc/\(.*\)/.*,\1,g' | while read pid; do sudo kill $pid; done
#
# Set size to be 12000 on singularity-hub
#
# Changelog
# ---------
# 2.3
# - fresh versions of the packages
# - freezing using WiP nd_freeze to the yesterday's day
# 2.2
# - fresh annex with patched git to avoid "Out of memory, getdelim failed" bug
# - added some tools useful for debugging (gdb)
# - additional mountpoints (/scratch)
# - (note) pip is not installed on purpose, use the one within virtualenv(s)
# 2.x
# - switch to stretch
# - TODO make reproducible
# - bids-validator from 0.22
# - Added ants, convert3d
#
# TODOs
# -----
# - package bids-validator
BootStrap: debootstrap
OSVersion: stretch
MirrorURL: http://http.debian.net/debian/
#MirrorURL: http://smaug.datalad.org:3142/debian/
# so if image is executed we just enter the environment
%runscript
v=`python -c "import json; f='/.singularity.d/labels.json'; print(json.load(open(f)).get('SINGULARITY_IMAGE_VERSION', '0.0.unknown'))"`; \
echo "Welcome to the NeuroDebian v $v (Debian stretch) environment"
echo "Please source /etc/fsl/fsl.sh if you need FSL, /etc/afni/afni.sh if you need AFNI"
/bin/bash
%setup
set -eu
export | grep SING
echo "Setting up the environment"
#apt-get update
#apt-get -y install python
v=`git describe --tags --match sing-\* | sed -e 's,^sing-,,g'`; \
python -c "import json, os; f='$SINGULARITY_ROOTFS/.singularity.d/labels.json'; j=json.load(open(f)) if os.path.exists(f) else {}; j['SINGULARITY_IMAGE_VERSION']='$v' or '0.0.unknown'; json.dump(j, open(f,'w'),indent=2)"
chmod a+r "$SINGULARITY_ROOTFS/.singularity.d/labels.json"
%post
echo "Configuring the environment"
sed -i -e 's, main$, main contrib non-free,g' /etc/apt/sources.list
# For build-dep
# sed -i -e 's,^deb \(.*\),deb \1\ndeb-src \1,g' /etc/apt/sources.list
apt-get update -q
apt-get -y install -q --no-install-recommends wget
wget -q -O /var/tmp/nd_freeze -nc https://raw.githubusercontent.com/neurodebian/neurodebian/2e1dab5cdf9021d59812d0fd00d4293e650c48e6/tools/nd_freeze
bash /var/tmp/nd_freeze 20180819T11:00:00Z
apt-get update -q
apt-get -y install eatmydata
eatmydata apt-get -y install vim wget strace gdb valgrind time ncdu gnupg curl procps
# eatmydata apt-get -y build-dep git
wget -q -O/tmp/nd-configurerepo https://raw.githubusercontent.com/neurodebian/neurodebian/4d26c8f30433145009aa3f74516da12f560a5a13/tools/nd-configurerepo
bash /tmp/nd-configurerepo
chmod a+r -R /etc/apt
eatmydata apt-get -y install git git-annex-standalone datalad python-nipype virtualenv dcm2niix python-dcmstack python-configparser python-funcsigs python-pytest connectome-workbench python-mvpa2 python-nilearn fsl-core fsl-atlases fsl-first-data mricron afni fsleyes ants convert3d heudiconv
# for bids-validator
curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
eatmydata apt-get install -y nodejs
npm install -g bids-validator@0.22.0
chmod a+rX -R /usr/lib/node_modules/
chmod a+rX -R /etc/apt/sources.list.d
# cleaning /tmp that thoroughly might have side effects
# rm -rf /tmp/* /var/tmp/*
rm -rf /tmp/npm-* /tmp/nd-config* /tmp/startup* /var/tmp/npm-*
apt-get clean
# and wipe out apt lists since not to be used RW for further tuning
# find /var/lib/apt/lists/ -type f -delete
# /usr/bin/find /var/lib/apt/lists/ -type f -name \*Packages\* -o -name \*Contents\*
# complicates later interrogation - thus disabled
# Create some additional bind mount directories present on various compute boxes we have
# access to, to ease deployment
mkdir -p /afs /inbox /ihome /opt /data /backup /apps /srv /scratch /idata
chmod a+rX /afs /inbox /ihome /opt /data /backup /apps /srv /scratch /idata
......@@ -2,7 +2,11 @@
#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
#ex: set sts=4 ts=4 sw=4 et:
# Depends: apt (assumed to be present), python, wget
# Depends:
# assumes to be present:
# apt
# installs if needed if NEURODEBIAN_INSTALL=1:
# python, wget, gnupg, dirmngr (when older gnupg)
# Recommends: netselect
# play safe
......@@ -545,6 +549,14 @@ fi
# Figure out if key needs to be imported (if ran within package,
# should already be there due to neurodebian-archive-keyring package)
# Ideas taken from neurodebian-docker setup to guarantee correct apt-key functioning
# gnupg needed by apt-key might not yet be installed
assure_command_from_package gpg gnupg 1
# Ubuntu includes "gnupg" (not "gnupg2", but still 2.x), but not dirmngr, and gnupg 2.x requires dirmngr
# so, if we're not running gnupg 1.x, explicitly install dirmngr too
gpg --version | grep -q '^gpg (GnuPG) 1\.' || assure_command_from_package dirmngr dirmngr 1
if LANG=C eval $ae_sudo apt-key export $nd_key_id 2>&1 1>/dev/null | grep -qe "nothing exported"; then
print_verbose 1 "Fetching the key from the server"
eval_dry apt-key adv --recv-keys --keyserver pgp.mit.edu $nd_key_id
......@@ -573,7 +585,7 @@ if [ "$ae_update" = "1" ]; then
error 5 "Update failed with exit code $?."
}
else
eval_dry apt-get update
eval_dry apt-get update # --no-allow-insecure-repositories
fi
else
print_verbose 1 "apt-get update was not run. Please run to take an effect of changes"
......
......@@ -139,7 +139,7 @@ additional components will be installed." \
FALSE "Emacs" "emacs" \
FALSE "Graphics (e.g. GIMP, Inscape)" "gimp,inkscape,svgtune,xzgv,vym" \
FALSE "Octave" "qtoctave" \
FALSE "OpenOffice.org" "openoffice.org" \
FALSE "LibreOffice" "libreoffice" \
FALSE "PyMVPA Tutorial" "$ipython,python-mvpa2,python-mvpa2-doc,python-mvpa2-tutorialdata,spyder" \
FALSE "Python: Electrophysiology & Modeling" "python-brian,python-pynn,python-pyentropy,stimfit" \
FALSE "Python: Neuroimaging" "$ipython,spyder,nipy-suite,python-mvpa2" \
......
......@@ -16,7 +16,9 @@ releases = {
'lenny': 'Debian GNU/Linux 5.0 (lenny)',
'squeeze': 'Debian GNU/Linux 6.0 (squeeze)',
'wheezy': 'Debian GNU/Linux 7.0 (wheezy)',
'jessie': 'Debian testing (jessie)',
'jessie': 'Debian GNU/Linux 8.0 (jessie)',
'stretch': 'Debian GNU/Linux 9.0 (stretch)',
'buster': 'Debian testing (buster)',
'sid': 'Debian unstable (sid)',
'hardy': 'Ubuntu 08.04 LTS "Hardy Heron" (hardy)',
'jaunty': 'Ubuntu 09.04 "Jaunty Jackalope" (jaunty)',
......@@ -30,6 +32,13 @@ releases = {
'raring': 'Ubuntu 13.04 "Raring Ringtail" (raring)',
'saucy': 'Ubuntu 13.10 "Saucy Salamander" (saucy)',
'trusty': 'Ubuntu 14.04 "Trusty Tahr" (trusty)',
'utopic': 'Ubuntu 14.10 "Utopic Unicorn" (utopic)',
'vivid': 'Ubuntu 15.04 "Vivid Vervet" (vivid)',
'wily': 'Ubuntu 15.10 "Wily Werewolf" (wily)',
'xenial': 'Ubuntu 16.04 "Xenial Xerus" (xenial)',
'yakkety': 'Ubuntu 16.10 "Yakkety Yak" (yakkety)',
'zesty': 'Ubuntu 17.04 "Zesty Zapus" (zesty)',
'artful': 'Ubuntu 17.10 "Artful Aardvark" (artful)',
}
......
This diff is collapsed.
#!/bin/bash
# Temporary scratch directory to save generated files.
TMP_DIR=""
TOOLS_PATH="${PWD}/${0%/*}/.."
EXIT_CODE=0
# There is tee calls to log output, but if docker run fails, we need to know
set -o pipefail
# Run common commands for each test at start of tests
#
# Parameters
# ----------
# distro : string
# Valid values are "debain" or "ubuntu"
# release : string
# Code name for distro release. (e.g. jessie)
# date : string
# Date at which the snapshot of the OS is wanted
# keep_sources : string (Optional)
# Pass the string "KEEP_SOURCES" to run nd_freeze with the --keep-original-apt-sources switch
#
function test_setup {
distro=$1
release=$2
date=$3
keep_sources=${4:-no}
if [ "$keep_sources" = "no" ]; then
keep_sources_param=""
else
keep_sources_param="--keep-original-apt-sources"
fi
TMP_DIR=$(mktemp -d)
echo "INFO: TMP_DIR=$TMP_DIR distro=$distro release=$release date=$date keep_sources=$keep_sources"
docker pull ${distro}:${release}
cmd="docker run -it --rm -v $TOOLS_PATH:/tests -v $TMP_DIR:/temp ${distro}:${release} /bin/bash -c '
/tests/nd_freeze $keep_sources_param $date
if [ -f /etc/apt/sources.list ]; then
cp /etc/apt/sources.list /temp/sources.list
fi
if [ -f /etc/apt/sources.list.original ]; then
cp /etc/apt/sources.list.original /temp/sources.list.original
fi
if [ -f /etc/apt/sources.list.d/neurodebian.sources.list ]; then
cp /etc/apt/sources.list.d/neurodebian.sources.list /temp/neurodebian.sources.list
fi
if [ -f /etc/apt/sources.list.d/neurodebian.sources.list.original ]; then
cp /etc/apt/sources.list.d/neurodebian.sources.list.original /temp/neurodebian.sources.list.original
fi
if [ -f /etc/apt/sources.list.d/snapshots.sources.list ]; then
cp /etc/apt/sources.list.d/snapshots.sources.list /temp/snapshots.sources.list
fi
'"
eval "$cmd" | tee $TMP_DIR/stdout
ret=$?
if [ "$ret" != "0" ]; then
echo "ERROR: docker run failed. Exit code was $ret"
echo "Full command was: $cmd"
EXIT_CODE=$(($EXIT_CODE + 1))
fi
}
# Run commands that are run at the end of each test run.
function test_teardown {
echo "INFO: removing TMP_DIR: ${TMP_DIR}"
rm -rf $TMP_DIR
}
# Test to see if a line is in a given file.
#
# Parameters
# ----------
# file : string
# File to search for the given string
#
# line : string
# Line the we expect to be in the file.
#
# match_mode : string
# Enter "partial_match" if the line given is not a complete line but a line fragment
#
function assert_line_in_file {
file=$1
line=$2
match_mode=${3:-full}
if [ "$match_mode" = "full" ]; then
match_param="x"
else
match_param=""
fi
filepath="$TMP_DIR/$file"
if grep -Fq$match_param "$line" "$filepath"; then
echo "SUCCESS: line '$line' found"
else
echo -n "FAIL: File $filepath did not contain: $line. "
if [ -e "$filepath" ]; then
echo "Full file content:"
cat "$filepath"
else
echo "File $filepath does not exist."
fi
EXIT_CODE=$(($EXIT_CODE + 1))
fi
}
echo "[ Test basic operation ]"
test_setup "neurodebian" "jessie" "7/27/2017"
assert_line_in_file "neurodebian.sources.list" "# deb http://neuro.debian.net/debian jessie main"
assert_line_in_file "neurodebian.sources.list" "# deb http://neuro.debian.net/debian data main"
assert_line_in_file "neurodebian.sources.list.original" "deb http://neuro.debian.net/debian jessie main"
assert_line_in_file "neurodebian.sources.list.original" "deb http://neuro.debian.net/debian data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20170727T050508Z/ jessie main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian/20170727T040550Z/ jessie-updates main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20170727T050508Z/ data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian/20170727T040550Z/ jessie main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian-security/20170727T203455Z/ jessie/updates main"
assert_line_in_file "sources.list" "# deb http://deb.debian.org/debian jessie main"
assert_line_in_file "sources.list" "# deb http://security.debian.org" "partial_match"
assert_line_in_file "sources.list" "# deb http://deb.debian.org/debian jessie-updates main"
assert_line_in_file "sources.list.original" "deb http://deb.debian.org/debian jessie main"
assert_line_in_file "sources.list.original" "deb http://security.debian.org" "partial_match"
assert_line_in_file "sources.list.original" "deb http://deb.debian.org/debian jessie-updates main"
test_teardown
echo "[ Test handling of a different release ]"
test_setup "neurodebian" "wheezy" "1/2/2018"
assert_line_in_file "neurodebian.sources.list" "# deb http://neuro.debian.net/debian wheezy main"
assert_line_in_file "neurodebian.sources.list" "# deb http://neuro.debian.net/debian data main"
assert_line_in_file "neurodebian.sources.list.original" "deb http://neuro.debian.net/debian wheezy main"
assert_line_in_file "neurodebian.sources.list.original" "deb http://neuro.debian.net/debian data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20180102T060503Z/ wheezy main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian/20180102T055258Z/ wheezy-updates main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20180102T060503Z/ data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian/20180102T055258Z/ wheezy main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian-security/20180103T134828Z/ wheezy/updates main"
assert_line_in_file "sources.list" "# deb http://deb.debian.org/debian wheezy main"
assert_line_in_file "sources.list" "# deb http://deb.debian.org/debian wheezy-updates main"
assert_line_in_file "sources.list" "# deb http://security.debian.org" "partial_match"
assert_line_in_file "sources.list.original" "deb http://deb.debian.org/debian wheezy main"
assert_line_in_file "sources.list.original" "deb http://deb.debian.org/debian wheezy-updates main"
assert_line_in_file "sources.list.original" "deb http://security.debian.org" "partial_match"
test_teardown
echo "[ Test handling of request for a snapshot in the future ]"
test_setup "neurodebian" "jessie" "20280827"
assert_line_in_file "neurodebian.sources.list" "deb http://neuro.debian.net/debian jessie main"
assert_line_in_file "sources.list" "deb http://deb.debian.org/debian jessie main"
assert_line_in_file "sources.list" "deb http://deb.debian.org/debian jessie-updates main"
assert_line_in_file "sources.list" "deb http://security.debian.org" "partial_match"
assert_line_in_file "stdout" 'INFO: WARNING: User specified time (20280827T000000Z) later than latest snapshot available' "partial_match"
test_teardown
echo "[ Test Ubuntu release ]"
test_setup "neurodebian" "xenial" "12/15/2017"
assert_line_in_file "neurodebian.sources.list" "# deb http://neuro.debian.net/debian xenial main"
assert_line_in_file "neurodebian.sources.list" "# deb http://neuro.debian.net/debian data main"
assert_line_in_file "neurodebian.sources.list.original" "deb http://neuro.debian.net/debian xenial main"
assert_line_in_file "neurodebian.sources.list.original" "deb http://neuro.debian.net/debian data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20171215T060503Z/ xenial main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20171215T060503Z/ data main"
assert_line_in_file "sources.list" "deb http://archive.ubuntu.com/ubuntu/ xenial main restricted"
assert_line_in_file "sources.list" "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted"
assert_line_in_file "sources.list" "deb http://security.ubuntu.com/ubuntu/ xenial-security universe"
test_teardown
echo "[ Test --keep-original-apt-sources switch ]"
test_setup "neurodebian" "jessie" "7/27/2017" "KEEP_SOURCES"
assert_line_in_file "neurodebian.sources.list" "deb http://neuro.debian.net/debian jessie main"
assert_line_in_file "neurodebian.sources.list" "deb http://neuro.debian.net/debian data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20170727T050508Z/ jessie main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian/20170727T040550Z/ jessie-updates main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20170727T050508Z/ data main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian/20170727T040550Z/ jessie main"
assert_line_in_file "snapshots.sources.list" "deb http://snapshot.debian.org/archive/debian-security/20170727T203455Z/ jessie/updates main"
assert_line_in_file "sources.list" "deb http://deb.debian.org/debian jessie main"
assert_line_in_file "sources.list" "deb http://security.debian.org" "partial_match"
assert_line_in_file "sources.list" "deb http://deb.debian.org/debian jessie-updates main"
test_teardown
exit $EXIT_CODE