Commit 7048e758 authored by Austin Blatt's avatar Austin Blatt

Merge 6.0.x into master

parents ea41e321 b47ed485
*
!project.clj
!src
!resources
!documentation
!docker/puppetdb
......@@ -22,7 +22,7 @@ services:
aliases:
- &run-core-and-ext-tests |
set -e
jdk="$(ext/travisci/jdk-from-spec "$PDB_TEST")"
jdk="$(ext/bin/jdk-from-spec "$PDB_TEST")"
jdkver="${jdk##*jdk}"
ext/travisci/prep-os-essentials-for "$PDB_TEST"
case "$OSTYPE" in
......@@ -52,12 +52,13 @@ aliases:
pgver="$(ext/travisci/prefixed-ref-from-spec "$PDB_TEST" pg-)"
ext/bin/test-config --set pgver "$pgver"
ext/bin/test-config --set pgport 34335
ext/bin/check-spec-env "$PDB_TEST"
ext/bin/boxed-core-tests -- lein test
ext/bin/run-external-tests
- &run-integration-tests |
set -e
jdk="$(ext/travisci/jdk-from-spec "$PDB_TEST")"
jdk="$(ext/bin/jdk-from-spec "$PDB_TEST")"
jdkver="${jdk##*jdk}"
ext/travisci/prep-os-essentials-for "$PDB_TEST"
case "$OSTYPE" in
......@@ -93,11 +94,13 @@ aliases:
ext/bin/test-config --set puppetserver-ref "$server"
PDB_TEST_RICH_DATA="$(ext/travisci/spec-includes "$PDB_TEST" rich)"
export PDB_TEST_RICH_DATA
ext/bin/check-spec-env "$PDB_TEST"
ext/bin/boxed-integration-tests -- lein test :integration
- &run-spec-tests |
set -e
puppet_ref="$(ext/travisci/prefixed-ref-from-spec "$PDB_TEST" pup-)"
ext/bin/check-spec-env "$PDB_TEST"
ext/bin/run-rspec-tests "$puppet_ref"
- &run-docker-tests |
......@@ -212,7 +215,11 @@ on_success: ext/travisci/on-success
notifications:
email: false
slack:
secure: IJU0YgGYbKgM7NupaOmE2BYra2mNx7+e5vAYNL+5oaRXolbHCyg0WzfFWilhMK3KEi8oIMKXR4ZzoUZLAqeOQzX7nnsLqC3wjyDHCgxtp4O+5GNKyeLN4ItoI1f2d6qyiiBPkHgVPuLhG3yyQ+wD0dMc9vSYmxfoazqe9HD/9UE=
template:
- "<%{compare_url}|%{commit_subject}> | %{author}"
- "%{repository_slug} %{branch} | <%{build_url}|#%{build_number}> %{result} in %{elapsed_time}"
rooms:
secure: IJU0YgGYbKgM7NupaOmE2BYra2mNx7+e5vAYNL+5oaRXolbHCyg0WzfFWilhMK3KEi8oIMKXR4ZzoUZLAqeOQzX7nnsLqC3wjyDHCgxtp4O+5GNKyeLN4ItoI1f2d6qyiiBPkHgVPuLhG3yyQ+wD0dMc9vSYmxfoazqe9HD/9UE=
cache:
directories:
......
......@@ -28,7 +28,7 @@ else
endif
build: prep
@docker build --pull --build-arg vcs_ref=$(vcs_ref) --build-arg build_date=$(build_date) --build-arg version=$(version) --file puppetdb/$(dockerfile) --tag puppet/puppetdb:$(version) puppetdb
@docker build --pull --build-arg vcs_ref=$(vcs_ref) --build-arg build_date=$(build_date) --build-arg version=$(version) --file puppetdb/$(dockerfile) --tag puppet/puppetdb:$(version) ..
ifeq ($(IS_LATEST),true)
@docker tag puppet/puppetdb:$(version) puppet/puppetdb:latest
endif
......
......@@ -42,7 +42,7 @@ function Build-Container(
'--tag', "$Namespace/puppetdb:latest"
)
docker build $docker_args puppetdb
docker build $docker_args ..
Pop-Location
}
......
FROM ubuntu:16.04
FROM clojure:lein-alpine AS builder
RUN apk add --no-cache make
# Install only dependencies
WORKDIR /app
COPY project.clj /app/
COPY resources/puppetlabs/puppetdb/bootstrap.cfg \
/app/resources/puppetlabs/puppetdb/bootstrap.cfg
RUN lein with-profile uberjar deps
# Build uberjar -- see .dockerignore
COPY . /app
RUN lein with-profile uberjar uberjar
FROM openjdk:8-jre-alpine
RUN apk add --no-cache tini curl openssl
COPY --from=builder /app/target/puppetdb.jar /
ARG vcs_ref
ARG build_date
ARG version="6.0.0"
ENV PUPPETDB_VERSION="$version"
ENV DUMB_INIT_VERSION="1.2.1"
ENV UBUNTU_CODENAME="xenial"
ENV PUPPETDB_DATABASE_CONNECTION="//postgres:5432/puppetdb"
ENV PUPPETDB_USER=puppetdb
ENV PUPPETDB_PASSWORD=puppetdb
......@@ -17,7 +30,6 @@ ENV PUPPETDB_JAVA_ARGS="-Djava.net.preferIPv4Stack=true -Xms256m -Xmx256m"
# used by entrypoint to determine if puppetserver should be contacted for config
# set to false when container tests are run
ENV USE_PUPPETSERVER=true
ENV PATH="/opt/puppetlabs/server/bin:/opt/puppetlabs/puppet/bin:/opt/puppetlabs/bin:$PATH"
LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
org.label-schema.vendor="Puppet" \
......@@ -31,36 +43,38 @@ LABEL org.label-schema.maintainer="Puppet Release Team <release@puppet.com>" \
org.label-schema.schema-version="1.0" \
org.label-schema.dockerfile="/Dockerfile"
RUN apt-get update && \
apt-get install --no-install-recommends -y wget netcat lsb-release ca-certificates && \
wget https://apt.puppetlabs.com/puppet6-release-"$UBUNTU_CODENAME".deb && \
wget https://github.com/Yelp/dumb-init/releases/download/v"$DUMB_INIT_VERSION"/dumb-init_"$DUMB_INIT_VERSION"_amd64.deb && \
dpkg -i puppet6-release-"$UBUNTU_CODENAME".deb && \
dpkg -i dumb-init_"$DUMB_INIT_VERSION"_amd64.deb && \
rm puppet6-release-"$UBUNTU_CODENAME".deb dumb-init_"$DUMB_INIT_VERSION"_amd64.deb && \
apt-get update && \
apt-get install --no-install-recommends -y puppet-agent puppetdb="$PUPPETDB_VERSION"-1"$UBUNTU_CODENAME" && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Values from /etc/default/puppetdb
ENV JAVA_BIN="/usr/bin/java"
ENV USER="puppetdb"
ENV GROUP="puppetdb"
ENV INSTALL_DIR="/opt/puppetlabs/server/apps/puppetdb"
ENV CONFIG="/etc/puppetlabs/puppetdb/conf.d"
ENV BOOTSTRAP_CONFIG="/etc/puppetlabs/puppetdb/bootstrap.cfg"
ENV SERVICE_STOP_RETRIES=60
COPY docker/puppetdb/logging /etc/puppetlabs/puppetdb/logging/
COPY docker/puppetdb/conf.d /etc/puppetlabs/puppetdb/conf.d/
COPY resources/puppetlabs/puppetdb/bootstrap.cfg /etc/puppetlabs/puppetdb/
COPY resources/ext/config/logback.xml /etc/puppetlabs/puppetdb/
COPY resources/ext/config/request-logging.xml /etc/puppetlabs/puppetdb/
RUN mkdir -p /opt/puppetlabs/server/data/puppetdb
COPY puppetdb /etc/default/
COPY logging /etc/puppetlabs/puppetdb/logging
RUN addgroup $GROUP && adduser -S $USER -G $GROUP
RUN rm -fr /etc/puppetlabs/puppetdb/conf.d
COPY conf.d /etc/puppetlabs/puppetdb/conf.d
ADD https://raw.githubusercontent.com/puppetlabs/pupperware/b651119f16a1c18d5e9174c283a4e535d35a128a/shared/ssl.sh /ssl.sh
RUN chmod +x /ssl.sh
COPY docker/puppetdb/ssl-setup.sh /
RUN chmod +x /ssl-setup.sh
# Persist the agent SSL certificate.
VOLUME /etc/puppetlabs/puppet/ssl/
# /etc/puppetlabs/puppetdb/ssl is automatically populated from here and
# doesn't need a separate volume.
COPY docker-entrypoint.sh /
COPY docker/puppetdb/docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh
EXPOSE 8080 8081
ENTRYPOINT ["dumb-init", "/docker-entrypoint.sh"]
CMD ["foreground"]
ENTRYPOINT ["/sbin/tini", "-g", "--", "/docker-entrypoint.sh"]
CMD ["services"]
# The start-period is just a wild guess how long it takes PuppetDB to come
# up in the worst case. The other timing parameters are set so that it
......@@ -73,4 +87,4 @@ HEALTHCHECK --interval=10s --timeout=10s --retries=6 \
| grep -q '"state":"running"' \
|| exit 1
COPY Dockerfile /
COPY docker/puppetdb/Dockerfile /
#!/bin/bash
#!/bin/sh
master_running() {
status=$(curl --silent --fail --insecure "https://${PUPPETSERVER_HOSTNAME}:8140/status/v1/simple")
......@@ -6,8 +6,6 @@ master_running() {
}
PUPPETSERVER_HOSTNAME="${PUPPETSERVER_HOSTNAME:-puppet}"
/opt/puppetlabs/bin/puppet config set certname "$HOSTNAME"
/opt/puppetlabs/bin/puppet config set server "$PUPPETSERVER_HOSTNAME"
if [ ! -f "/etc/puppetlabs/puppet/ssl/certs/${HOSTNAME}.pem" ] && [ "$USE_PUPPETSERVER" = true ]; then
# if this is our first run, run puppet agent to get certs in place
......@@ -15,10 +13,12 @@ if [ ! -f "/etc/puppetlabs/puppet/ssl/certs/${HOSTNAME}.pem" ] && [ "$USE_PUPPET
sleep 1
done
set -e
/opt/puppetlabs/bin/puppet agent --verbose --onetime --no-daemonize --waitforcert 120
/ssl.sh
fi
if [ ! -d "/etc/puppetlabs/puppetdb/ssl" ] && [ "$USE_PUPPETSERVER" = true ]; then
/opt/puppetlabs/server/bin/puppetdb ssl-setup -f
/ssl-setup.sh -f
fi
exec /opt/puppetlabs/server/bin/puppetdb "$@"
exec java $PUPPETDB_JAVA_ARGS -cp /puppetdb.jar \
clojure.main -m puppetlabs.puppetdb.core "$@" \
-c /etc/puppetlabs/puppetdb/conf.d/
###########################################
# Init settings for puppetdb
###########################################
# Location of your Java binary (version 8 or higher)
JAVA_BIN="/usr/bin/java"
# Modify this if you'd like to change the memory allocation, enable JMX, etc
JAVA_ARGS="$PUPPETDB_JAVA_ARGS"
# These normally shouldn't need to be edited if using OS packages
USER="puppetdb"
GROUP="puppetdb"
INSTALL_DIR="/opt/puppetlabs/server/apps/puppetdb"
CONFIG="/etc/puppetlabs/puppetdb/conf.d"
BOOTSTRAP_CONFIG="/etc/puppetlabs/puppetdb/bootstrap.cfg"
SERVICE_STOP_RETRIES=60
# START_TIMEOUT can be set here to alter the default startup timeout in
# seconds. This is used in System-V style init scripts only, and will have no
# effect in systemd.
# START_TIMEOUT=300
This diff is collapsed.
......@@ -33,7 +33,27 @@ canonical: "/puppetdb/latest/release_notes.html"
- **Improved error handling for invalid or malformed timestamps.** If you passed an invalid or malformed timestamp in a PQL query, PuppetDB treated it as a `null`, giving back an unexpected query result.
[PDB-4015](https://tickets.puppetlabs.com/browse/PDB-4015)
## PuppetDB 6.0.1
## 6.0.2
### Improvements
- **Improved PostgreSQL support.** PuppetDB is now compatible with PostgreSQL version 10 and later. [PDB-3857](https://tickets.puppetlabs.com/browse/PDB-3857)
### Bug Fixes
- **(PE Only) PuppetDB no longer syncs reports that are older than the `report-ttl`, but have not yet been garbage collected.** PuppetDB would sync reports when it performed an initial garbage collection on startup, and then sync reports from its remote, which likely had not performed garbage collection as recently.
[PDB-4158](https://tickets.puppetlabs.com/browse/PDB-4158)
- **PuppetDB skips unnecessary work when ingesting commands.** PuppetDB wasn't pulling `producer-timestamp` out of the incoming query parameters for `submit` command requests. This caused PuppetDB to not have the necessary information to tombstone obsolete commands if multiple `store facts` or `store catalog` commands were submitted for the same `certname` while the earlier commands were still in the queue waiting to be processed.
[PDB-4177](https://tickets.puppetlabs.com/browse/PDB-4177)
- **Improved error handling for invalid or malformed timestamps.** If you passed an invalid or malformed timestamp in a PQL query, PuppetDB treated it as a `null`, giving back an unexpected query result.
[PDB-4015](https://tickets.puppetlabs.com/browse/PDB-4015)
- **PuppetDB no longer causes PostgreSQL to create large amounts of temporary files during garbage collection.** This issue caused PostgreSQL's log to flood if the `log_temp_files` option was set to a small enough value. [PDB-3924](https://tickets.puppetlabs.com/browse/PDB-3924)
### Security
- We've blacklisted the jackson-databind dependency to resolve several security issues. [PDB-4236](https://tickets.puppetlabs.com/browse/PDB-4236)
## 6.0.1
PuppetDB 6.0.1 is a new feature and bug-fix release.
......@@ -100,6 +120,26 @@ when the minimum successful submissions have been met. ([PDB-4020](https://ticke
Austin Blatt, Britt Gresham, Charlie Sharpsteen, Garrett Guillotte, Jarret Lavallee
Molly Waggett, Morgan Rhodes, Rob Browning, and Zachary Kent
## 5.2.7
### Improvements
- **Improved PostgreSQL support.** PuppetDB is now compatible with PostgreSQL version 10 and later. [PDB-3857](https://tickets.puppetlabs.com/browse/PDB-3857)
### Bug Fixes
- **(PE Only) PuppetDB no longer syncs reports that are older than the `report-ttl`, but have not yet been garbage collected.** PuppetDB would sync reports when it performed an initial garbage collection on startup, and then sync reports from its remote, which likely had not performed garbage collection as recently.
[PDB-4158](https://tickets.puppetlabs.com/browse/PDB-4158)
- **PuppetDB skips unnecessary work when ingesting commands.** PuppetDB wasn't pulling `producer-timestamp` out of the incoming query parameters for `submit` command requests. This caused PuppetDB to not have the necessary information to tombstone obsolete commands if multiple `store facts` or `store catalog` commands were submitted for the same `certname` while the earlier commands were still in the queue waiting to be processed.
[PDB-4177](https://tickets.puppetlabs.com/browse/PDB-4177)
- **Improved error handling for invalid or malformed timestamps.** If you passed an invalid or malformed timestamp in a PQL query, PuppetDB treated it as a `null`, giving back an unexpected query result.
[PDB-4015](https://tickets.puppetlabs.com/browse/PDB-4015)
- **PuppetDB no longer causes PostgreSQL to create large amounts of temporary files during garbage collection.** This issue caused PostgreSQL's log to flood if the `log_temp_files` option was set to a small enough value. [PDB-3924](https://tickets.puppetlabs.com/browse/PDB-3924)
### Security
- We've blacklisted the jackson-databind dependency to resolve several security issues. [PDB-4236](https://tickets.puppetlabs.com/browse/PDB-4236)
## 5.2.6
PuppetDB 5.2.6 is a security, new feature, and bug-fix release.
......
#!/usr/bin/env bash
set -ueo pipefail
usage()
{
echo "Usage: $(basename $0) SPEC"
}
misuse()
{
usage 1>&2
exit 2
}
verify-jdk() {
local spec="$1"
echo spec: [$spec]
local actual_jdk expected_jdk
expected_jdk="$(ext/bin/jdk-from-spec "$spec")"
actual_jdk="$(ext/bin/jdk-info --print spec)"
if test "$expected_jdk" != "$actual_jdk"; then
printf "JDK in path %q is not what PDB_TEST specifies: %q\n" \
"$actual_jdk" "$expected_jdk" 1>&2
exit 2
fi
}
test $# -ne 1 && misuse
spec="$1"
test_kind=''
for item in ${spec//// }; do # change / to ' '
echo item: "[$item]"
case "$item" in
core+ext|int|rspec) test_kind="$item" ;;
esac
done
if test -z "$test_kind"; then
echo "Unable to find a test category in $spec" 1>&2
exit 2
fi
case "$test_kind" in
core+ext|int)
verify-jdk "$spec"
;;
rspec)
;;
*)
echo "Unexpected test category: $test_kind" 1>&2
exit 2
;;
esac
#!/usr/bin/env bash
set -ueo pipefail
usage()
{
echo "Usage: $(basename $0) <--print (version|major|spec)>..."
}
misuse()
{
usage 1>&2
exit 2
}
extract-major() {
local ver="$1"
local ver_x="${ver%%.*}"
if test "$ver_x" -ge 10; then
echo "$ver_x"
else
echo "$ver" | sed -E 's/[0-9]+\.([0-9]+).*/\1/'
fi
}
print-info() {
local what="$1"
local ver ver_out
# java -version prints the requested info to stderr...
ver_out=$(java -version 2>&1 1>/dev/null)
# Looks like the third field of the first line always has a
# quoted version string. Some variants:
# java version "1.8.0_172"
# openjdk version "1.8.0_171"
# openjdk version "10.0.2" 2018-07-17
# openjdk version "10.0.2-adoptopenjdk" 2018-07-17
ver=$(echo "$ver_out" | head -1 | cut -d' ' -f 3 | sed -E 's/^"(.+)"$/\1/')
case "$what" in
version) echo "$ver" ;;
major) echo "$(extract-major "$ver")" ;;
spec)
local maj_ver
maj_ver=$(extract-major "$ver")
if echo "$ver_out" | grep -qiF openjdk; then
echo "openjdk$maj_ver"
else
echo "oraclejdk$maj_ver"
fi
;;
*) misuse ;;
esac
}
if test $# -eq 0; then
misuse
fi
while test $# -gt 0; do
case "$1" in
--print)
shift
test $# -gt 0 || misuse
what="$1"
shift
case "$what" in
version|major|spec)
print-info "$what"
;;
*) misuse ;;
esac
;;
*)
misuse
esac
done
......@@ -7,6 +7,23 @@ misuse() { usage 1>&2; exit 2; }
test $# -eq 0 || misuse
jdkver=$(ext/bin/jdk-info --print major)
expected_help_warnings=0
expected_version_warnings=0
case "$jdkver" in
8)
;;
10)
expected_help_warnings=0
expected_version_warnings=535
;;
*)
echo "JDK version '$jdkver' is not supported" 1>&2
exit 3
;;
esac
tmpdir="$(mktemp -d "test-top-level-cli-XXXXXX")"
tmpdir="$(cd "$tmpdir" && pwd)"
trap "$(printf 'rm -rf %q' "$tmpdir")" EXIT
......@@ -24,7 +41,9 @@ cat "$tmpdir/out" "$tmpdir/err"
test "$rc" -eq 0
grep -F 'Available subcommands:' "$tmpdir/out"
grep -E 'Display version information' "$tmpdir/out"
test $(wc -c < "$tmpdir/err") -eq 0
# FIXME: this should and will be 0 once we fix the pos-int?, dynapath,
# etc. replacement warnings.
test $(wc -c < "$tmpdir/err") -eq $expected_help_warnings
rc=0
./pdb version 1>"$tmpdir/out" 2>"$tmpdir/err" || rc=$?
......@@ -32,4 +51,6 @@ cat "$tmpdir/out" "$tmpdir/err"
test "$rc" -eq 0
grep -E '^version=' "$tmpdir/out"
grep -E '^target_schema_version=' "$tmpdir/out"
test $(wc -c < "$tmpdir/err") -eq 0
# FIXME: this should and will be 0 once we fix the pos-int?, dynapath,
# etc. replacement warnings.
test $(wc -c < "$tmpdir/err") -eq $expected_version_warnings
......@@ -14,28 +14,33 @@ misuse() {
test $# -eq 1 || misuse
spec="$1"
jdk="$(ext/travisci/jdk-from-spec "$spec")"
jdk="$(ext/bin/jdk-from-spec "$spec")"
jdkver="${jdk##*jdk}"
pgver="$(ext/travisci/prefixed-ref-from-spec "$spec" pg-)"
case "$OSTYPE" in
darwin*)
# brew produced some HOMEBREW_LOGS related error on the first
# run but said that "everything should be fine" if you try
# again, so we do that...
brew install ruby || true
brew install ruby
brew install bash
brew install postgresql@"$pgver"
brew tap AdoptOpenJDK/openjdk
case "$jdkver" in
8|10)
# Install AdoptOpenJDK 11, we will use this for its cacert
brew cask install https://raw.githubusercontent.com/Homebrew/homebrew-cask/636d8f0d1afce664f47620b46571e42b01c93d8c/Casks/adoptopenjdk.rb
cacert_path=/Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home/lib/security/cacerts
brew cask install adoptopenjdk11
cacert_path=/Library/Java/JavaVirtualMachines/adoptopenjdk-11.0.1.jdk/Contents/Home/lib/security/cacerts
if test ! -f "$cacert_path"; then
echo "The cacerts file did not exist at '$cacert_path'" 1>&2
exit 3
fi
brew tap AdoptOpenJDK/openjdk
brew cask install "adopt$jdk"
old_cacert_path=
......
(def pdb-version "6.1.1-SNAPSHOT")
(def clj-parent-version "2.5.0")
(def clj-parent-version "2.6.0")
(defn true-in-env? [x]
(#{"true" "yes" "1"} (System/getenv x)))
......
#!/bin/bash
#
# MAINTAINERS:
# This file was copied and modified to run in the
# Docker container here: docker/puppetdb/ssl-setup.sh
#
# If you change this .erb file, consider changing the
# .sh version of this file as well.
ssl_command="puppetdb ssl-setup"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment