Commit a96c53e8 authored by Ken Barber's avatar Ken Barber

PDB-279 Sanitize report imports

Previously we had a bug PDB-85 that caused our exports on 1.5.x to fail. This
has been fixed, but alas people are trying to import those broken dumps into
1.6.x and finding it doesn't work.

This patch sanitizes our imports by only using select keys from the reports
model and dropping everything else.
Signed-off-by: default avatarKen Barber <>
parent 734247ad
......@@ -10,6 +10,7 @@
[com.puppetlabs.http :as pl-http]
[com.puppetlabs.archive :as archive]
[com.puppetlabs.cheshire :as json]
[com.puppetlabs.puppetdb.reports :as reports]
[ :as io])
(:import [com.puppetlabs.archive TarGzReader]
[org.apache.commons.compress.archivers.tar TarArchiveEntry])
......@@ -44,9 +45,9 @@
(integer? command-version)
(string? catalog-payload)]}
(let [result (command/submit-command-via-http!
puppetdb-host puppetdb-port
puppetdb-host puppetdb-port
(command-names :replace-catalog) command-version
(when-not (= pl-http/status-ok (:status result))
(log/error result))))
......@@ -58,10 +59,13 @@
(integer? puppetdb-port)
(integer? command-version)
(string? report-payload)]}
(let [result (command/submit-command-via-http!
(let [payload (-> report-payload
result (command/submit-command-via-http!
puppetdb-host puppetdb-port
(command-names :store-report) command-version
(json/parse-string report-payload))]
(when-not (= pl-http/status-ok (:status result))
(log/error result))))
......@@ -23,6 +23,10 @@
:type :string }
(def report-fields
"Report fields"
(keys (:fields Report)))
(defmodel ResourceEvent
{:status :string
:timestamp :datetime
......@@ -44,6 +48,10 @@
:type :coll }
(def resource-event-fields
"Resource event fields"
(keys (:fields ResourceEvent)))
(def v2-new-event-fields [:file :line])
(defn validate-and-add-v2-event-field!
......@@ -87,3 +95,24 @@
(format "Containment path should only contain strings: '%s'"
(resource-event :containment-path))))))
(defn sanitize-events
"This function takes an array of events and santizes them, ensuring only
valid keys are returned."
{:pre [(coll? events)]
:post [(coll? %)]}
(let [valid-keys (map name resource-event-fields)]
(for [event events]
(select-keys event valid-keys))))
(defn sanitize-report
"This function takes a report and sanitizes it, ensuring only valid data
is left over."
{:pre [(map? payload)]
:post [(map? %)]}
(let [valid-keys (map name report-fields)]
(-> payload
(select-keys valid-keys)
(update-in ["resource-events"] sanitize-events))))
......@@ -43,3 +43,30 @@
IllegalArgumentException #":timestamp should be Datetime"
(validate! 2 (assoc-in report [:resource-events 0 :timestamp] "foo")))))))
(deftest test-sanitize-events
(testing "ensure extraneous keys are removed"
(let [test-data {"containment-path"
"Postgresql_psql[create extension pg_stat_statements on puppetdb]"],
"new-value" "CREATE EXTENSION pg_stat_statements",
"command changed '' to 'CREATE EXTENSION pg_stat_statements'",
"old-value" nil,
"status" "success",
"line" 16,
"property" "command",
"timestamp" "2014-01-09T17:52:56.795Z",
"resource-type" "Postgresql_psql",
"resource-title" "create extension pg_stat_statements on puppetdb",
"file" "/etc/puppet/modules/my_pg/manifests/extension.pp"
"extradata" "foo"}
santized (sanitize-events [test-data])
expected [(dissoc test-data "extradata")]]
(= santized expected))))
(deftest test-sanitize-report
(testing "no action on valid reports"
(let [test-data (clojure.walk/stringify-keys (:basic reports))]
(= (sanitize-report test-data) test-data))))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment