Commit 69374588 authored by Russell Mull's avatar Russell Mull

(maint) Support iterative acceptance test development

Add support for iterating on acceptance tests more quickly by allowing a
developer to re-run them against an already created VM.

- Add PUPPETDB_SKIP_INSTALL and BEAKER_NO_PROVISION environment variables
  to the Rakefile
- Add a beaker configuration designated for running these tests 
  (ec2-west-dev)
- Add the beaker:first_run and beaker:rerun rake tasks
- Use a patched version of beaker that works on OS X hosts
- Document how this all works
parent fb6ea5d3
......@@ -19,3 +19,6 @@ Vagrantfile
*.tar.gz
.blimpy.d
version
junit
tmp
vendor
......@@ -57,8 +57,12 @@ group :acceptance do
#use the specified version
gem 'beaker', *location_for(beaker_version)
else
#use the pinned version
gem 'beaker', '~>2.4'
# use the pinned version
# We're pinning to a a prerelease version of beaker that fixes errors when
# running with an OS X host. We only need to do this until a new version is
# released with the fix included.
gem 'beaker', :git => 'git://github.com/puppetlabs/beaker.git', :ref => '3e2e1e6d3e0234612c3a6af7dbfac06c797676f4'
end
# This forces google-api-client to not download retirable 2.0.0 which lacks
# ruby 1.9.x support.
......
......@@ -3,15 +3,16 @@ require 'rake'
RAKE_ROOT = File.dirname(__FILE__)
def run_beaker(test_files)
config = ENV["BEAKER_CONFIG"] || "vbox-el6-64mda"
config = ENV["BEAKER_CONFIG"] || "ec2-west-dev"
options = ENV["BEAKER_OPTIONS"] || "postgres"
preserve_hosts = ENV["BEAKER_PRESERVE_HOSTS"] || "never"
no_provision = ENV["BEAKER_NO_PROVISION"] == "true" ? true : false
color = ENV["BEAKER_COLOR"] == "false" ? false : true
xml = ENV["BEAKER_XML"] == "true" ? true : false
type = ENV["BEAKER_TYPE"] || "git"
rake_root = File.dirname(__FILE__)
keyfile = ENV["BEAKER_KEYFILE"] || nil
beaker = "beaker " +
beaker = "bundle exec beaker " +
"-c '#{RAKE_ROOT}/acceptance/config/#{config}.cfg' " +
"--type #{type} " +
"--debug " +
......@@ -22,6 +23,7 @@ def run_beaker(test_files)
beaker += " --no-color" unless color
beaker += " --xml" if xml
beaker += " --no-provision" if no_provision
sh beaker
end
......@@ -38,4 +40,25 @@ namespace :beaker do
args.with_defaults(:test_files => 'acceptance/performance/')
run_beaker(args[:test_files])
end
desc "Run beaker based acceptance tests, leaving VMs in place for later runs"
task :first_run, [:test_files] do |t, args|
args.with_defaults(:test_files => 'acceptance/tests/')
ENV["BEAKER_PRESERVE_HOSTS"] = "always"
run_beaker(args[:test_files])
end
desc "Re-run beaker based acceptance tests, using previously provisioned VMs"
task :rerun, [:test_files] do |t, args|
args.with_defaults(:test_files => 'acceptance/tests/')
ENV["PUPPETDB_SKIP_PRESUITE_PROVISIONING"] = "true"
ENV["BEAKER_NO_PROVISION"] = "true"
ENV["BEAKER_PRESERVE_HOSTS"] = "always"
run_beaker(args[:test_files])
end
desc "List your VMs in ec2"
task :list_vms do
sh 'aws ec2 describe-instances --filters "Name=key-name,Values=Beaker-${USER}*" --query "Reservations[*].Instances[*].[InstanceId, State.Name, PublicIpAddress]" --output table'
end
end
HOSTS:
debian7-64-1:
roles:
- master
- database
- dashboard
- agent
vmname: debian-7-amd64-west
platform: debian-7-amd64
amisize: c3.large
hypervisor: ec2
snapshot: foss
subnet_id: subnet-4e768a39
vpc_id: vpc-3ea7605b
# ip: 54.186.214.143
CONFIG:
nfs_server: none
consoleport: 443
......@@ -54,6 +54,12 @@ module PuppetDBExtensions
"'purge packages and perform exhaustive cleanup after run'",
"PUPPETDB_PURGE_AFTER_RUN", :false)
skip_presuite_provisioning =
get_option_value(options[:puppetdb_skip_presuite_provisioning],
[:true, :false],
"'skip installation steps",
"PUPPETDB_SKIP_PRESUITE_PROVISIONING", :false)
package_build_host =
get_option_value(options[:puppetdb_package_build_host],
nil,
......@@ -112,6 +118,7 @@ module PuppetDBExtensions
:repo_hiera => puppetdb_repo_hiera,
:repo_facter => puppetdb_repo_facter,
:git_ref => puppetdb_git_ref,
:skip_presuite_provisioning => skip_presuite_provisioning == :true,
}
pp_config = PP.pp(@config, "")
......
unless (options[:vmrun])
unless (options[:vmrun]) or (ENV['PUPPETDB_SKIP_PRESUITE_PROVISIONING'])
step "Clean up configuration files on master" do
on master, "rm -rf /etc/puppet/routes.yaml"
end
......
step "Flushing iptables chains" do
hosts.each do |host|
on host, "iptables -F INPUT -t filter"
on host, "iptables -F FORWARD -t filter"
unless (test_config[:skip_presuite_provisioning])
step "Flushing iptables chains" do
hosts.each do |host|
on host, "iptables -F INPUT -t filter"
on host, "iptables -F FORWARD -t filter"
end
end
end
......@@ -14,8 +14,10 @@ def upgrade_pkgs_on_host(host, os)
end
end
step "Upgrade each host" do
hosts.each do |host|
upgrade_pkgs_on_host(host, test_config[:os_families][host.name])
unless (test_config[:skip_presuite_provisioning])
step "Upgrade each host" do
hosts.each do |host|
upgrade_pkgs_on_host(host, test_config[:os_families][host.name])
end
end
end
......@@ -55,8 +55,10 @@ gpgcheck=1
end
end
step "Install Puppet Labs repositories" do
hosts.each do |host|
initialize_repo_on_host(host, test_config[:os_families][host.name])
unless (test_config[:skip_presuite_provisioning])
step "Install Puppet Labs repositories" do
hosts.each do |host|
initialize_repo_on_host(host, test_config[:os_families][host.name])
end
end
end
test_name "Install Puppet" do
step "Install Puppet" do
install_puppet
unless (test_config[:skip_presuite_provisioning])
step "Install Puppet" do
install_puppet
end
end
step "Populate facts from each host" do
......
step "Run an agent to create the SSL certs" do
databases.each do |database|
with_puppet_running_on(master, {'master' => {'autosign' => 'true', 'trace' => 'true'}}) do
run_agent_on(database, "--test")
unless (test_config[:skip_presuite_provisioning])
step "Run an agent to create the SSL certs" do
databases.each do |database|
with_puppet_running_on(master, {'master' => {'autosign' => 'true', 'trace' => 'true'}}) do
run_agent_on(database, "--test")
end
end
end
end
step "Install other dependencies on database" do
databases.each do |database|
os = test_config[:os_families][database.name]
db_facts = facts(database.name)
unless (test_config[:skip_presuite_provisioning])
step "Install other dependencies on database" do
databases.each do |database|
os = test_config[:os_families][database.name]
db_facts = facts(database.name)
use_our_jdk = ((db_facts["osfamily"] == "Debian") and
(db_facts["operatingsystemmajrelease"] == "6" or
db_facts["operatingsystemrelease"] == "10.04"))
use_our_jdk = ((db_facts["osfamily"] == "Debian") and
(db_facts["operatingsystemmajrelease"] == "6" or
db_facts["operatingsystemrelease"] == "10.04"))
# Install our JDK repository with a JDK 7 for Debian 6 and Ubuntu 10.04
# and install the oracle jdk
if use_our_jdk then
create_remote_file database, '/etc/apt/sources.list.d/jpkg.list', <<-REPO
# Oracle JDK Packages
deb http://s3-us-west-2.amazonaws.com/puppetdb-jdk/jpkg/ pljdk main
REPO
# Import GPG key
on database, "gpg --keyserver keys.gnupg.net --recv-keys B8615A77BBBFA17C"
on database, "gpg -a --export B8615A77BBBFA17C | apt-key add -"
on database, "apt-get update"
end
# Install our JDK repository with a JDK 7 for Debian 6 and Ubuntu 10.04
# and install the oracle jdk
if use_our_jdk then
create_remote_file database, '/etc/apt/sources.list.d/jpkg.list', <<-REPO
# Oracle JDK Packages
deb http://s3-us-west-2.amazonaws.com/puppetdb-jdk/jpkg/ pljdk main
REPO
# Import GPG key
on database, "gpg --keyserver keys.gnupg.net --recv-keys B8615A77BBBFA17C"
on database, "gpg -a --export B8615A77BBBFA17C | apt-key add -"
on database, "apt-get update"
end
if test_config[:install_type] == :git then
case os
when :debian
if use_our_jdk then
# Use our jdk
on database, "apt-get install -y --force-yes oracle-j2sdk1.7 rake unzip"
if test_config[:install_type] == :git then
case os
when :debian
if use_our_jdk then
# Use our jdk
on database, "apt-get install -y --force-yes oracle-j2sdk1.7 rake unzip"
else
# Other debians have a JDK 7 already, just use that
on database, "apt-get install -y --force-yes openjdk-7-jre-headless rake unzip"
end
when :redhat
on database, "yum install -y java-1.7.0-openjdk rubygem-rake unzip"
when :fedora
on database, "yum install -y java-1.7.0-openjdk rubygem-rake unzip"
else
# Other debians have a JDK 7 already, just use that
on database, "apt-get install -y --force-yes openjdk-7-jre-headless rake unzip"
raise ArgumentError, "Unsupported OS '#{os}'"
end
when :redhat
on database, "yum install -y java-1.7.0-openjdk rubygem-rake unzip"
when :fedora
on database, "yum install -y java-1.7.0-openjdk rubygem-rake unzip"
else
raise ArgumentError, "Unsupported OS '#{os}'"
end
step "Install lein on the PuppetDB server" do
which_result = on database, "which lein", :acceptable_exit_codes => [0,1]
needs_lein = which_result.exit_code == 1
if (needs_lein)
on database, "curl --tlsv1 -Lk https://raw.github.com/technomancy/leiningen/stable/bin/lein -o /usr/local/bin/lein"
on database, "chmod +x /usr/local/bin/lein"
on database, "LEIN_ROOT=true lein"
step "Install lein on the PuppetDB server" do
which_result = on database, "which lein", :acceptable_exit_codes => [0,1]
needs_lein = which_result.exit_code == 1
if (needs_lein)
on database, "curl --tlsv1 -Lk https://raw.github.com/technomancy/leiningen/stable/bin/lein -o /usr/local/bin/lein"
on database, "chmod +x /usr/local/bin/lein"
on database, "LEIN_ROOT=true lein"
end
end
end
end
end
step "Install rubygems and sqlite3 on master" do
os = test_config[:os_families][master.name]
step "Install rubygems and sqlite3 on master" do
os = test_config[:os_families][master.name]
case os
when :redhat
case master['platform']
when /^el-5/
on master, "yum install -y rubygems sqlite-devel rubygem-activerecord ruby-devel.x86_64"
on master, "gem install sqlite3"
when /^el-6/
on master, "yum install -y rubygems ruby-sqlite3 rubygem-activerecord"
else
# EL7 very much matches what Fedora 20 uses
case os
when :redhat
case master['platform']
when /^el-5/
on master, "yum install -y rubygems sqlite-devel rubygem-activerecord ruby-devel.x86_64"
on master, "gem install sqlite3"
when /^el-6/
on master, "yum install -y rubygems ruby-sqlite3 rubygem-activerecord"
else
# EL7 very much matches what Fedora 20 uses
on master, "yum install -y rubygems rubygem-sqlite3"
on master, "gem install activerecord -v 3.2.17 --no-ri --no-rdoc -V --backtrace"
end
when :fedora
# This was really set with Fedora 20 in mind, later versions might differ
on master, "yum install -y rubygems rubygem-sqlite3"
on master, "gem install activerecord -v 3.2.17 --no-ri --no-rdoc -V --backtrace"
end
when :fedora
# This was really set with Fedora 20 in mind, later versions might differ
on master, "yum install -y rubygems rubygem-sqlite3"
on master, "gem install activerecord -v 3.2.17 --no-ri --no-rdoc -V --backtrace"
when :debian
case master['platform']
when /^ubuntu-10.04/
# Ubuntu 10.04 has rubygems 1.3.5 which is known to not be reliable, so therefore
# we skip.
when :debian
case master['platform']
when /^ubuntu-10.04/
# Ubuntu 10.04 has rubygems 1.3.5 which is known to not be reliable, so therefore
# we skip.
else
on master, "apt-get install -y rubygems ruby-dev libsqlite3-dev"
# this is to get around the activesupport dependency on Ruby 1.9.3 for
# Ubuntu 12.04. We can remove it when we drop support for 1.8.7.
on master, "gem install i18n -v 0.6.11"
on master, "gem install activerecord -v 3.2.17 --no-ri --no-rdoc -V --backtrace"
on master, "gem install sqlite3 -v 1.3.9 --no-ri --no-rdoc -V --backtrace"
end
else
on master, "apt-get install -y rubygems ruby-dev libsqlite3-dev"
# this is to get around the activesupport dependency on Ruby 1.9.3 for
# Ubuntu 12.04. We can remove it when we drop support for 1.8.7.
on master, "gem install i18n -v 0.6.11"
on master, "gem install activerecord -v 3.2.17 --no-ri --no-rdoc -V --backtrace"
on master, "gem install sqlite3 -v 1.3.9 --no-ri --no-rdoc -V --backtrace"
raise ArgumentError, "Unsupported OS: '#{os}'"
end
else
raise ArgumentError, "Unsupported OS: '#{os}'"
end
# Make sure there isn't a gemrc file, because that could ruin our day.
on master, "rm -f ~/.gemrc"
# Make sure there isn't a gemrc file, because that could ruin our day.
on master, "rm -f ~/.gemrc"
end
end
end
......@@ -2,6 +2,8 @@ require 'beaker/dsl/install_utils'
extend Beaker::DSL::InstallUtils
step "Install the puppetdb module and dependencies" do
on databases, "puppet module install puppetlabs/puppetdb"
unless (test_config[:skip_presuite_provisioning])
step "Install the puppetdb module and dependencies" do
on databases, "puppet module install puppetlabs/puppetdb"
end
end
step "Create a fake entry in /etc/hosts to prevent test nodes from checking in with dujour" do
manifest_content = <<-EOS
host { "updates.puppetlabs.com":
ip => '127.0.0.1',
ensure => 'present',
}
EOS
databases.each do |database|
manifest_path = database.tmpfile("puppetdb_manifest.pp")
create_remote_file(database, manifest_path, manifest_content)
on database, puppet_apply("#{manifest_path}")
unless (test_config[:skip_presuite_provisioning])
step "Create a fake entry in /etc/hosts to prevent test nodes from checking in with dujour" do
manifest_content = <<-EOS
host { "updates.puppetlabs.com":
ip => '127.0.0.1',
ensure => 'present',
}
EOS
databases.each do |database|
manifest_path = database.tmpfile("puppetdb_manifest.pp")
create_remote_file(database, manifest_path, manifest_content)
on database, puppet_apply("#{manifest_path}")
end
end
end
# We skip this step entirely unless we are running in :upgrade mode.
if (test_config[:install_mode] == :upgrade)
if (test_config[:install_mode] == :upgrade and not test_config[:skip_presuite_provisioning])
step "Install most recent released PuppetDB on the PuppetDB server for upgrade test" do
databases.each do |database|
install_puppetdb(database, test_config[:database], 'latest')
......
if (test_config[:install_type] == :package)
if (test_config[:install_type] == :package and not test_config[:skip_presuite_provisioning])
databases.each do |database|
os = test_config[:os_families][database.name]
db_facts = facts(database.name)
......
......@@ -5,6 +5,7 @@ step "Install development build of PuppetDB on the PuppetDB server" do
case test_config[:install_type]
when :git
Log.notify("Install puppetdb from source")
Log.error database
if (test_config[:database] == :postgres)
install_postgres(database)
......
Acceptance Tests
----------------
PuppetDB uses the [Beaker](https://github.com/puppetlabs/beaker) acceptance
testing framework. We run acceptance tests on a matrix of machine and database
configurations before we merge new code into our stable or master branches, but
it can be useful for a variety of reasons to run them yourself.
The current recommended way of running acceptance tests is via EC2. Other
methods should be possible, as Beaker supports a wide variety of hypervisors.
EC2 Setup
---------
* Create ~/.fog with these contents. Note that 'aws access key id' is the thing
otherwise called an AWS access key. It's not your user id.
:default:
:aws_access_key_id: <your AWS access key>
:aws_secret_access_key: <your AWS secret key>
* The included configuration files in `acceptance/config` refer to resources
(security groups, VPCs, etc) that exist in the Puppet Labs AWS account. If
you're using your own AWS account, you'll have to create the appropriate
resources and modify the appropriate configuration file to refer to them.
Running the Tests
-----------------
* If you previously munged your host file as described below, first remove the
IP address entry.
* Do a normal acceptance test run the first time, so you get a fully provisioned VM to work with.
rake "beaker:first_run[acceptance/tests/some/test.rb]"
* For now, you have to modify the host file you're using to include the IP
addresses for the VMs that were provisioned the first time. You should be able
to find these near the top of the console output. You could also ask the
hypervisor for help:
rake beaker:list_vms
Take the IP address of the VM that was created for you and put it in the
appropriate beaker hosts file. If you didn't specify one, the default is
`acceptance/config/ec2-west-dev.cfg`. It has a commented-out IP address field,
you should be able to uncomment it and put in the IP of your running VM.
* For subsequent runs, you can reuse the already-provisioned VM for a quicker
turnaround time.
rake "beaker:rerun[acceptance/tests/some/test.rb]"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment