Commit 9bd489e7 authored by Rob Browning's avatar Rob Browning

(PDB-3917) Stop mangling project.clj via sed for integration tests

Move the install-gems code to src-gems and manage it via an
install-gems profile so that the puppetserver dependency doesn't
always have to be known (install_gems.clj refers to puppetserver
namespaces and so can't be compiled without a puppetserver
dependency).

Making the puppetserver dependency optional is important because until
we actually check out the ./puppetserver tree, we don't know what the
version is, i.e. for 5.3.x it might be 5.3.7-SNAPSHOT.

Have configure-int-tests store the puppetserver test dependency
version in ext/test-conf/puppetserver-dep, and adjust everything to
read the version from there when the file exists, or behave reasonably
when it doesn't.

Previously, not knowing the real depdendency version hasn't been an
issue because we've set puppetserver-version in the project.clj to
some fixed version, and then mangled the project.clj via sed to adjust
it during integration testing.
parent a34dfde9
# Leiningen
/target/
/target-gems/
/pom.xml
# Acceptance tests
......
......@@ -82,22 +82,18 @@ set `PDB_TEST_PRESERVE_DB_ON_FAIL` to true:
$ PDB_TEST_KEEP_DB_ON_FAIL=true lein test
To run the integration tests, you'll need to ensure you have a
suitable version of Ruby available, and then install the relevant gems
by running something like this from the top level of the source tree:
suitable version of Ruby and Bundler available, and then run:
$ bundle install --path "$(pwd)/vendor"
`bundle` will store the gems in `vendor/` and will remember that
location for future invocations. You're welcome to choose a different
`--path`, but note that the current integration tests will adulterate
the content, so you may not want to share it.
$ ext/bin/configure-int-tests
To finish setting up the integration tests, invoke:
You can request specific versions of puppet or puppetserver with the
`PUPPET_VERSION` and `PUPPETSERVER_VERSION` environment variables:
$ lein install-gems
$ PUPPET_VERSION=5.3.x PUPPETSERVER_VERSION=5.1.x \
ext/bin/configure-int-tests
at which point you should be able to run the tests by specifying the
`:integration` selector:
After configuration, you should be able to run the tests by specifying
the `:integration` selector:
$ lein test :integration
......
#!/usr/bin/env bash
set -euo pipefail
script_home="$(cd "$(dirname "$0")" && pwd)"
top="$script_home/../.."
cd "$top" # Always run from the top level of the tree
usage() {
echo "Usage: $(basename "$0") [REF]"
}
lein-pprint() {
lein with-profile dev,ci pprint "$@" | sed -e 's/^"//' -e 's/"$//'
}
PDB_NO_PUPPETSERVER_INSTALL="${PDB_NO_PUPPETSERVER_INSTALL:-}"
if test -z "$(type -t bundler)"; then
cat 2>&1 <<-EOS
Ruby bundler does not appear to be available.
Please install it via "gem install --user-install bundler" or similar.
EOS
exit 2
fi
set -x
rm -f ext/test-conf/puppetserver-requested ext/test-conf/puppetserver-dep
case "$#" in
0)
pupsrv_ref="$(cat ext/test-conf/puppetserver-default)"
;;
1)
pupsrv_ref="$1"
echo "$pupsrv_ref" > ext/test-conf/puppetserver-requested
;;
*)
usage 1>&2
exit 2
esac
if test -d puppetserver; then
git -C puppetserver checkout "$pupsrv_ref"
git -C puppetserver clean -fdx
else
git clone --depth 10 -b "$pupsrv_ref" https://github.com/puppetlabs/puppetserver
fi
(export PUPPETSERVER_HEAP_SIZE=1G
cd puppetserver
# If we're not testing against the git tree (say in jenkins), don't install.
if test -z "$PDB_NO_PUPPETSERVER_INSTALL"; then
lein install
fi
dep_ver="$(lein-pprint :version)"
echo "$dep_ver" > "$top/ext/test-conf/puppetserver-dep")
lein install-gems
......@@ -2,6 +2,10 @@
set -euo pipefail
script_home="$(cd "$(dirname "$0")" && pwd)"
top="$script_home/../.."
cd "$top" # Always run from the top level of the tree
if test -z "$(type -t bundler)"; then
cat 2>&1 <<-EOS
Ruby bundler does not appear to be available.
......@@ -14,40 +18,30 @@ set -x
export PUPPETSERVER_HEAP_SIZE=1G
PDB_NO_PUPPETSERVER_INSTALL="${PDB_NO_PUPPETSERVER_INSTALL:-}"
PUPPET_VERSION="${PUPPET_VERSION:-master}"
PUPPETSERVER_VERSION="${PUPPETSERVER_VERSION:-master}"
# Update a single dependency in a leiningen project.clj file.
update_dependency_var() {
local file="$1"
local varname="$2"
local new_version="$3"
SED_ADDRESS="(def $varname"
SED_REGEX="\".*\""
SED_REPLACEMENT="\"$new_version\""
SED_COMMAND="s|$SED_REGEX|$SED_REPLACEMENT|"
set -x
sed -i -e "/$SED_ADDRESS/ $SED_COMMAND" "$file"
}
lein-pprint() {
lein with-profile dev,ci pprint "$@" | sed -e 's/^"//' -e 's/"$//'
}
git clone https://github.com/puppetlabs/puppetserver
cd puppetserver
git checkout "$PUPPETSERVER_VERSION"
lein install
MAVEN_VER="$(lein-pprint :version)"
echo "$MAVEN_VER"
cd ..
if test -d puppetserver; then
git -C puppetserver checkout "$PUPPETSERVER_VERSION"
git -C puppetserver clean -fdx
else
git clone --depth 10 -b "$PUPPETSERVER_VERSION" https://github.com/puppetlabs/puppetserver
fi
update_dependency_var project.clj puppetserver-version "$MAVEN_VER"
(cd puppetserver
if test -z "$PDB_NO_PUPPETSERVER_INSTALL"; then
lein install
fi
dep_ver="$(lein-pprint :version)"
mkdir -p "$top/ext/test-conf"
echo "$dep_ver" > "$top/ext/test-conf/puppetserver-dep")
bundle install --without acceptance --path "$(pwd)/vendor"
bundle install --without acceptance --path vendor/bundle
lein install-gems
# Symlink vendor/puppet to the git tree bundler checked out. Use a
......
......@@ -32,8 +32,6 @@ lein --version
lein clean
lein test
# We're going to run the integration tests, so we need both flavors of gems installed
NO_ACCEPTANCE=true bundle install --path vendor/bundle --without acceptance
lein install-gems
NO_ACCEPTANCE=true exec lein test :integration
export NO_ACCEPTANCE=true
PDB_NO_PUPPETSERVER_INSTALL=true ext/bin/configure-int-tests
exec lein test :integration
(def pdb-version "5.1.6-SNAPSHOT")
(def puppetserver-version "5.1.0")
(def clj-parent-version "1.4.3")
(defn pdb-run-sh [& args]
......@@ -23,6 +22,40 @@
(def need-permgen?
(= "1.7" (System/getProperty "java.specification.version")))
;; See the integration tests section in documentation/CONTRIBUTING.md.
(def puppetserver-test-dep-ver
(some-> (try
(slurp (str "ext/test-conf/puppetserver-dep"))
(catch java.io.FileNotFoundException ex
(binding [*out* *err*]
(println "puppetserver test depdency unconfigured (ignoring)"))
nil))
clojure.string/trim))
(def puppetserver-test-deps
(when puppetserver-test-dep-ver
`[[puppetlabs/puppetserver ~puppetserver-test-dep-ver]
[puppetlabs/puppetserver ~puppetserver-test-dep-ver :classifier "test"]]))
(def pdb-dev-deps
(concat
'[[ring-mock]
[puppetlabs/trapperkeeper :classifier "test"]
[puppetlabs/kitchensink :classifier "test"]
[puppetlabs/trapperkeeper-webserver-jetty9 :classifier "test"]
[org.flatland/ordered "1.5.3"]
[org.clojure/test.check "0.9.0"]
[com.gfredericks/test.chuck "0.2.7" :exclusions
[clj-time com.andrewmcveigh/cljs-time instaparse joda-time org.clojure/clojure]]
[environ "1.0.2"]
[riddley "0.1.12"]
[io.forward/yaml "1.0.5"]
;; Only needed for :integration tests
[puppetlabs/trapperkeeper-authorization nil]
[puppetlabs/trapperkeeper-filesystem-watcher nil]]
puppetserver-test-deps))
;; Don't use lein :clean-targets so that we don't have to repeat
;; ourselves, given that we need to remove some protected files, and
;; in addition, metadata like {:protect false} doesn't appear to
......@@ -40,6 +73,7 @@
"resources/puppetlabs/puppetdb/Messages_eo$1.class"
"resources/puppetlabs/puppetdb/Messages_eo.class"
"target"
"target-gems"
"test-resources/puppetserver/ssl/certificate_requests"
"test-resources/puppetserver/ssl/private"])
......@@ -175,23 +209,7 @@
:classifiers [["test" :testutils]]
:profiles {:dev {:resource-paths ["test-resources"],
:dependencies [[ring-mock]
[puppetlabs/trapperkeeper :classifier "test"]
[puppetlabs/kitchensink :classifier "test"]
[puppetlabs/trapperkeeper-webserver-jetty9 :classifier "test"]
[puppetlabs/puppetserver ~puppetserver-version]
[puppetlabs/puppetserver ~puppetserver-version :classifier "test"]
[org.flatland/ordered "1.5.3"]
[org.clojure/test.check "0.9.0"]
[com.gfredericks/test.chuck "0.2.7" :exclusions
[clj-time com.andrewmcveigh/cljs-time instaparse joda-time org.clojure/clojure]]
[environ "1.0.2"]
[riddley "0.1.12"]
[io.forward/yaml "1.0.5"]
;; Only needed for :integration tests
[puppetlabs/trapperkeeper-authorization nil]
[puppetlabs/trapperkeeper-filesystem-watcher nil]]
:dependencies ~pdb-dev-deps
:injections [(do
(require 'schema.core)
(schema.core/set-fn-validation! true))]}
......@@ -220,6 +238,9 @@
:name "puppetdb"
:plugins [[puppetlabs/lein-ezbake "1.8.1"]]}
:testutils {:source-paths ^:replace ["test"]}
:install-gems {:source-paths ^:replace ["src-gems"]
:target-path "target-gems"
:dependencies ~puppetserver-test-deps}
:ci {:plugins [[lein-pprint "1.1.1"]]}
:test {:jvm-opts ~(if need-permgen?
;; integration tests cycle jruby a lot, which chews through permgen
......@@ -243,9 +264,11 @@
(if (map? prev) [new prev] (conj prev new)))
#(spit %1 (pr-str %2))]}
:aliases {"gem" ["trampoline" "run" "-m" "puppetlabs.puppetserver.cli.gem"
:aliases {"gem" ["with-profie" "install-gems"
"trampoline" "run" "-m" "puppetlabs.puppetserver.cli.gem"
"--config" "./test-resources/puppetserver/puppetserver.conf"]
"install-gems" ["trampoline" "run" "-m" "puppetlabs.puppetdb.integration.install-gems"
"install-gems" ["with-profile" "install-gems"
"trampoline" "run" "-m" "puppetlabs.puppetdb.integration.install-gems"
"--config" "./test-resources/puppetserver/puppetserver.conf"]
"clean" ~(pdb-run-clean pdb-clean-paths)
"distclean" ~(pdb-run-clean pdb-distclean-paths)})
......@@ -3,8 +3,7 @@
[puppetlabs.puppetserver.cli.subcommand :as cli]
[puppetlabs.services.jruby.jruby-puppet-core :as jruby-puppet-core]
[puppetlabs.services.jruby-pool-manager.jruby-core :as jruby-core]
[clojure.string :as string]
[environ.core :refer [env]]))
[clojure.string :as string]))
(defn gem-run!
[config & args]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment