Commit 155d1e3f authored by Wayne's avatar Wayne Committed by Spencer McElmurry

(QENG-2198) Move PuppetDB acceptance over to vmpooler.

* Add sqa-utils gem.
* Rewrite `acceptance/setup/pre_suite/80_add_dev_repo.rb` to use
  `install_puppetlabs_dev_repo` instead of the special stuff it was doing.
* Add logic to `ext/jenkins/beaker-tests.sh` that will allow PR and merge tests
  to run without making changes to ci-job-configs for vmpooler (those changes
  are coming though!)
* In `ext/jenkins/packaging.sh` provide `PUPPETDB_PACKAGE_BUILD_VERSION` instead
  of `PACKAGE_BUILD_VERSION` to avoid having to convert to that variable in
  `ext/jenkins/beaker-tests.sh`.
* Remove S3 packaging/cleanup stuff.
* Set default beaker type to `aio`. This isn't really necessary, but the test
  suite doesn't work with `foss`, at least not on stable branch.
* BEAKER_CONFIG should take a filename. Same applies to BEAKER_OPTIONS. This
  change matches up with the way we treat variables of the same name in CI.

(cherry picked from commit 1698d86ae521d769d67f6e179a0371b3388b3e2c)
parent 97158e65
......@@ -61,4 +61,6 @@ if ENV['NO_ACCEPTANCE'] != 'true'
gem 'beaker', '~> 3.4'
end
end
gem 'beaker-hostgenerator', '0.8.2'
gem 'beaker-abs', *location_for(ENV['BEAKER_ABS_VERSION'] || '~> 0.2')
end
......@@ -3,26 +3,29 @@ require 'rake'
RAKE_ROOT = File.dirname(__FILE__)
def run_beaker(test_files)
config = ENV["BEAKER_CONFIG"] || "ec2-west-dev"
config = ENV["BEAKER_CONFIG"] || "acceptance/config/vmpooler.cfg"
options = ENV["BEAKER_OPTIONS"] || "acceptance/options/postgres.rb"
preserve_hosts = ENV["BEAKER_PRESERVE_HOSTS"] || "never"
no_provision = ENV["BEAKER_NO_PROVISION"] == "true" ? true : false
color = ENV["BEAKER_COLOR"] == "false" ? false : true
xml = ENV["BEAKER_XML"] == "true" ? true : false
type = ENV["BEAKER_TYPE"] || "git"
type = ENV["BEAKER_TYPE"] || "aio"
keyfile = ENV["BEAKER_KEYFILE"] || nil
collect_perf_data = ENV["BEAKER_COLLECT_PERF_DATA"] || nil
beaker = "bundle exec beaker " +
"-c '#{RAKE_ROOT}/acceptance/config/#{config}.cfg' " +
"-c '#{config}' " +
"--type #{type} " +
"--debug " +
"--tests " + test_files + " " +
"--options-file " + options + " " +
"--options-file '#{options}' " +
"--root-keys " +
"--preserve-hosts #{preserve_hosts}"
beaker += " --keyfile #{keyfile}" if keyfile
beaker += " --no-color" unless color
beaker += " --xml" if xml
beaker += " --collect-perf-data #{collect_perf_data}" if collect_perf_data
beaker += " --no-provision" if no_provision
sh beaker
......
HOSTS:
debian7-64-1:
pe_dir:
pe_ver:
pe_upgrade_dir:
pe_upgrade_ver:
hypervisor: vmpooler
platform: debian-7-amd64
template: debian-7-x86_64
roles:
- agent
- master
- database
- dashboard
CONFIG:
nfs_server: none
consoleport: 443
pooling_api: http://vmpooler.delivery.puppetlabs.net/
\ No newline at end of file
install_opts = options.merge( { :dev_builds_repos => ["PC1"] })
repo_config_dir = 'tmp/repo_configs'
if (test_config[:install_type] == :package and not test_config[:skip_presuite_provisioning])
databases.each do |database|
os = test_config[:os_families][database.name]
db_facts = facts(database.name)
sed_cmd = "sed 's/#{Regexp.escape(test_config[:package_build_host])}/#{Regexp.escape(test_config[:package_repo_host])}/'"
step "Add development repository on PuppetDB server" do
case os
when :debian
result = on database, "lsb_release -sc"
deb_flavor = result.stdout.chomp
apt_list_url = "#{test_config[:package_repo_url]}/repo_configs/deb/pl-puppetdb-#{test_config[:package_build_version]}-#{deb_flavor}.list"
apt_list_file_path = "/etc/apt/sources.list.d/puppetdb-prerelease.list"
on database, "curl \"#{apt_list_url}\" | #{sed_cmd} > #{apt_list_file_path}"
result = on database, "cat #{apt_list_file_path}"
Log.notify("APT LIST FILE CONTENTS:\n#{result.stdout}\n")
on database, "apt-get update"
when :redhat
el_version = db_facts["os"]["release"]["major"]
yum_repo_url = "#{test_config[:package_repo_url]}/repo_configs/rpm/pl-puppetdb-#{test_config[:package_build_version]}-el-#{el_version}-x86_64.repo"
yum_repo_file_path = "/etc/yum.repos.d/puppetlabs-prerelease.repo"
on database, "curl \"#{yum_repo_url}\" | #{sed_cmd} > #{yum_repo_file_path}"
result = on database, "cat #{yum_repo_file_path}"
Log.notify("Yum REPO DEFINITION:\n\n#{result.stdout}\n\n")
when :fedora
version = db_facts["os"]["release"]["full"]
yum_repo_url = "#{test_config[:package_repo_url]}/repo_configs/rpm/pl-puppetdb-#{test_config[:package_build_version]}-fedora-f#{version}-x86_64.repo"
yum_repo_file_path = "/etc/yum.repos.d/puppetlabs-prerelease.repo"
on database, "curl \"#{yum_repo_url}\" | #{sed_cmd} > #{yum_repo_file_path}"
result = on database, "cat #{yum_repo_file_path}"
Log.notify("Yum REPO DEFINITION:\n\n#{result.stdout}\n\n")
else
raise ArgumentError, "Unsupported OS '#{os}'"
end
end
install_puppetlabs_dev_repo database, 'puppetdb', test_config[:package_build_version],
repo_config_dir, install_opts
end
end
......@@ -11,22 +11,41 @@ echo "REPO_URL: ${REPO_URL}"
echo "PACKAGE_BUILD_VERSION: ${PACKAGE_BUILD_VERSION}"
echo "**********************************************"
set -x
# Beaker params
export BEAKER_COLOR=false
export BEAKER_XML=true
export PUPPETDB_INSTALL_TYPE=package
export PUPPETDB_USE_PROXIES=false
export BEAKER_project=PuppetDB
export BEAKER_department=eso-dept
export BEAKER_department=sre-dept
export BEAKER_PRESERVE_HOSTS=onfail
export BEAKER_COLLECT_PERF_DATA=aggressive
# S3 params
export PUPPETDB_PACKAGE_REPO_HOST="puppetdb-prerelease.s3.amazonaws.com"
export PUPPETDB_PACKAGE_REPO_URL="http://puppetdb-prerelease.s3.amazonaws.com/puppetdb/${PACKAGE_BUILD_VERSION}"
export PUPPETDB_PACKAGE_BUILD_VERSION=$PACKAGE_BUILD_VERSION
export BEAKER_CONFIG=$LAYOUT
export INSTALL_MODE=$INSTALL_TYPE
# Once the necessary PR (puppetlabs/ci-job-configs#2319) has been
# merged, this logic can be removed.
[ "$LAYOUT" = "ec2-west-el7-64mda-el7-64a" ] \
&& LAYOUT=centos7-64mda-64a
[ "$LAYOUT" = "ec2-west-el6-64mda-el6-64a" ] \
&& LAYOUT=centos6-64mda-64a
[ "$LAYOUT" = "ec2-west-ubuntu1204-64mda-64a" ] \
&& LAYOUT=ubuntu1204-64mda-64a
[ "$LAYOUT" = "ec2-west-ubuntu1404-64mda-64a" ] \
&& LAYOUT=ubuntu1404-64mda-64a
[ "$LAYOUT" = "ec2-west-ubuntu1604-64mda-64a" ] \
&& LAYOUT=ubuntu1604-64mda-64a
[ "$LAYOUT" = "ec2-west-debian7-64mda-64a" ] \
&& LAYOUT=debian7-64mda-64a
[ "$LAYOUT" = "ec2-west-el6-64mda-el5-64a-ubuntu1204-64a" ] \
&& LAYOUT=centos6-64mda-centos5-64a-ubuntu1204-64a
[ "$LAYOUT" = "ec2-west-debian7-64mda-fallback" ] \
&& LAYOUT=debian7-64mda-64d
[ "$LAYOUT" = "ec2-west-debian8-64mda-64a" ] \
&& LAYOUT=debian8-64mda-64a
export BEAKER_OPTIONS=acceptance/options/${PUPPETDB_DATABASE}.rb
export BEAKER_CONFIG=acceptance/hosts.cfg
bundle exec beaker-hostgenerator $LAYOUT --hypervisor abs > $BEAKER_CONFIG
set -x
# Now run our tests
bundle exec rake beaker:acceptance
#!/bin/bash
echo "**********************************************"
echo "PARAMS FROM UPSTREAM:"
echo ""
echo "PACKAGE_BUILD_VERSION: ${PACKAGE_BUILD_VERSION}
echo" "**********************************************"
REPO_DIR=/opt/jenkins-builds/puppetdb/$PACKAGE_BUILD_VERSION/repos
REPO_CONFIGS=/opt/jenkins-builds/puppetdb/$REF/repo_configs
NAME=puppetdb
BUCKET_NAME=${NAME}-prerelease
S3_BRANCH_PATH=s3://${BUCKET_NAME}/${NAME}/${PACKAGE_BUILD_VERSION}
set -x
# Set ship targets and build team for packaging repo
REPO_HOST=builds.delivery.puppetlabs.net
export REPO_DIR REPO_HOST
# Now rebuild the metadata
ssh $REPO_HOST <<PUBLISH_PACKAGES
set -e
set -x
echo "BUCKET_NAME IS: ${BUCKET_NAME}"
time s3cmd del --recursive ${S3_BRANCH_PATH}/
PUBLISH_PACKAGES
......@@ -11,63 +11,25 @@ echo "REPO_URL: ${REPO_URL}"
echo "**********************************************"
set -x
set -e
export COW="base-jessie-amd64.cow base-precise-amd64.cow base-trusty-amd64.cow base-wheezy-amd64.cow base-wily-amd64.cow base-xenial-amd64.cow"
export MOCK="pl-el-6-x86_64 pl-el-7-x86_64"
tmp_m2=$(pwd)/$(mktemp -d m2-local.XXXX)
set -e
lein update-in : assoc :local-repo "\"${tmp_m2}\"" -- install
lein update-in : assoc :local-repo "\"${tmp_m2}\"" -- deps
# The ci-voom profile has nexus credentials
lein update-in : assoc :local-repo "\"${tmp_m2}\"" -- with-profile user,ci-voom deploy
lein update-in : assoc :local-repo "\"${tmp_m2}\"" -- with-profile ezbake ezbake build
set +e
lein update-in : assoc :local-repo "\"${tmp_m2}\"" -- with-profile ezbake ezbake stage
pushd "target/staging"
PACKAGE_BUILD_VERSION=$(rake pl:print_build_param[ref] | tail -n 1)
rake package:bootstrap
rake pl:jenkins:uber_build[5]
cat > "${WORKSPACE}/puppetdb.packaging.props" <<PROPS
PACKAGE_BUILD_VERSION=${PACKAGE_BUILD_VERSION}
PUPPETDB_PACKAGE_BUILD_VERSION=$(rake pl:print_build_param[ref] | tail -n 1)
PROPS
popd
echo "**********************************************"
echo "NOW DO S3 MAGIC"
echo "**********************************************"
REPO_DIR=/opt/jenkins-builds/puppetdb/$PACKAGE_BUILD_VERSION/repos
REPO_CONFIGS=/opt/jenkins-builds/puppetdb/$PACKAGE_BUILD_VERSION/repo_configs
NAME=puppetdb
BUCKET_NAME=${NAME}-prerelease
S3_BRANCH_PATH=s3://${BUCKET_NAME}/${NAME}/${PACKAGE_BUILD_VERSION}
set -x
# Set ship targets and build team for packaging repo
REPO_HOST=builds.delivery.puppetlabs.net
export REPO_DIR REPO_HOST
# Now rebuild the metadata
# We add the `--follow-symlinks` flag to the apt metadata in preparation for
# the switch from reprepro to aptly. Aptly employs symlinks to maintain the
# current expected directory structure. As we cannot ship symlinks to s3, this
# flag will send over the resolved files.
ssh $REPO_HOST <<PUBLISH_PACKAGES
set -e
set -x
echo "BUCKET_NAME IS: ${BUCKET_NAME}"
time s3cmd --verbose --acl-public --delete-removed sync ${REPO_CONFIGS}/* ${S3_BRANCH_PATH}/repo_configs/
time s3cmd --verbose --acl-public --delete-removed --follow-symlinks sync ${REPO_DIR}/apt/{jessie,lucid,precise,trusty,wheezy,wily,xenial} ${S3_BRANCH_PATH}/repos/apt/
time s3cmd --verbose --acl-public --delete-removed sync ${REPO_DIR}/el/{5,6,7} ${S3_BRANCH_PATH}/repos/el/
time s3cmd --verbose --acl-public --delete-removed sync ${REPO_DIR}/fedora/f* ${S3_BRANCH_PATH}/repos/fedora/
PUBLISH_PACKAGES
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment