Commit ab4916b9 authored by Andrew Roetker's avatar Andrew Roetker

(PDB-1984) Move anonymization and export code to use v6 of reports

This commit moves the anonymization, export, and import code to use the
v6 wireformat of reports.
parent 146d6cac
......@@ -18,13 +18,15 @@
(contains? edge "target")
(contains? edge "relationship")))
(def resource-event-schema-str (utils/str-schema reports/resource-event-v5-wireformat-schema))
(def event-schema-str (utils/str-schema reports/event-wireformat-schema))
(def resource-schema-str (utils/str-schema (assoc reports/resource-wireformat-schema
:events [event-schema-str])))
(def metric-schema-str (utils/str-schema reports/metric-wireformat-schema))
(def log-schema-str (utils/str-schema reports/log-wireformat-schema))
(def report-schema-str (utils/str-schema (assoc reports/report-v5-wireformat-schema
:resource_events [resource-event-schema-str]
:metrics (s/maybe [metric-schema-str])
:logs (s/maybe [log-schema-str]))))
(def report-schema-str (utils/str-schema (assoc reports/report-wireformat-schema
:resources [resource-schema-str]
:metrics [metric-schema-str]
:logs [log-schema-str])))
(defn resource?
"Returns true if it looks like a resource"
......@@ -320,37 +322,42 @@
:post [(coll? %)]}
(map #(anonymize-catalog-resource % context config) resources))
(pls/defn-validated anonymize-resource-event :- resource-event-schema-str
"Anonymize a resource event from a report"
[event :- resource-event-schema-str
(defn anonymize-event
[{:strs [message property old_value new_value] :as event}
context config]
(let [newcontext {"node" (get context "node")
"title" (get event "resource_title")
"message" (get event "message")
"property_name" (get event "property")
"type" (get event "resource_type")
"file" (get event "file")
"line" (get event "line")}]
(let [newcontext (assoc context
"message" message
"property_name" property)]
(-> event
(update-in ["resource_title"] anonymize-leaf :title newcontext config)
(update-in ["message"] anonymize-leaf :message newcontext config)
(update-in ["property"] anonymize-leaf :parameter-name newcontext config)
(update-in ["new_value"]
anonymize-leaf :parameter-value
(assoc newcontext :parameter-value (get event "new_value")) config)
(update-in ["old_value"]
anonymize-leaf :parameter-value
(assoc newcontext :parameter-value (get event "old_value")) config)
(update-in ["resource_type"] anonymize-leaf :type newcontext config)
(update-in-nil ["file"] anonymize-leaf :file newcontext config)
(update-in-nil ["line"] anonymize-leaf :line newcontext config)
(update-in-nil ["containment_path"]
anonymize-containment-path newcontext config))))
(pls/defn-validated anonymize-metric
[metric :- metric-schema-str
context
config]
(update "message" anonymize-leaf :message newcontext config)
(update "property" anonymize-leaf :parameter-name newcontext config)
(update "new_value" anonymize-leaf :parameter-value (assoc newcontext :parameter-value new_value) config)
(update "old_value" anonymize-leaf :parameter-value (assoc newcontext :parameter-value old_value) config))))
(defn anonymize-events [events context config]
(map #(anonymize-event % context config) events))
(defn anonymize-report-resource
[resource context config]
(let [newcontext {"node" (get context "node")
"title" (get resource "resource_title")
"type" (get resource "resource_type")
"file" (get resource "file")
"line" (get resource "line")}]
(-> resource
(update "resource_title" anonymize-leaf :title newcontext config)
(update "resource_type" anonymize-leaf :type newcontext config)
(utils/update-when ["file"] anonymize-leaf :file newcontext config)
(utils/update-when ["line"] anonymize-leaf :line newcontext config)
(utils/update-when ["containment_path"] anonymize-containment-path newcontext config)
(update "events" anonymize-events newcontext config))))
(pls/defn-validated anonymize-report-resources :- [resource-schema-str]
[resources :- [resource-schema-str]
context config]
(map #(anonymize-report-resource % context config) resources))
(defn anonymize-metric [metric context config]
(if (= "time" (get metric "category"))
(update metric "name" #(anonymize-lowercase-type % context config))
metric))
......@@ -373,17 +380,7 @@
[logs :- [log-schema-str]
context
config]
(when logs
(map #(anonymize-log % context config) logs)))
(defn anonymize-resource-events
"Anonymize a collection of resource events from a report"
[events context config]
{:pre [(coll? events)]
:post [(coll? %)]}
(sort-by
#(mapv % ["timestamp" "resource_type" "resource_title" "property"])
(map #(anonymize-resource-event % context config) events)))
(map #(anonymize-log % context config) logs))
;; Primary entry points, for anonymizing catalogs and reports
......@@ -392,25 +389,25 @@
[config catalog]
{:pre [(catalog? catalog)]
:post [(catalog? %)]}
(let [context {"node" (get catalog ["certname"])}]
(let [context {"node" (get catalog "certname")}]
(-> catalog
(update-in ["resources"] anonymize-resources context config)
(update-in ["edges"] anonymize-edges context config)
(update-in ["certname"] anonymize-leaf :node context config)
(update-in ["transaction_uuid"] anonymize-leaf :transaction_uuid context config)
(update-in ["environment"] anonymize-leaf :environment context config))))
(update "resources" anonymize-catalog-resources context config)
(update "edges" anonymize-edges context config)
(update "certname" anonymize-leaf :node context config)
(update "transaction_uuid" anonymize-leaf :transaction_uuid context config)
(update "environment" anonymize-leaf :environment context config))))
(pls/defn-validated anonymize-report :- report-schema-str
"Anonymize a report"
[config report :- report-schema-str]
(let [context {"node" (get report "certname")}]
(-> report
(update-in ["certname"] anonymize-leaf :node context config)
(update-in ["resource_events"] anonymize-resource-events context config)
(update-in ["metrics"] anonymize-metrics context config)
(update-in ["logs"] anonymize-logs context config)
(update-in ["transaction_uuid"] anonymize-leaf :transaction_uuid context config)
(update-in ["environment"] anonymize-leaf :environment context config))))
(update "certname" anonymize-leaf :node context config)
(update "resources" anonymize-report-resources context config)
(update "metrics" anonymize-metrics context config)
(update "logs" anonymize-logs context config)
(update "transaction_uuid" anonymize-leaf :transaction_uuid context config)
(update "environment" anonymize-leaf :environment context config))))
(defn anonymize-fact-values
"Anonymizes fact names and values"
......
......@@ -24,7 +24,7 @@
;; version of the `catalog` endpoint... or even to query what the latest
;; version of a command is. We should improve that.
{:replace_catalog 6
:store_report 5
:store_report 6
:replace_facts 4})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
......@@ -90,7 +90,7 @@
:query->wire-fn catalogs/catalogs-query->wire-v6
:anonymize-fn anon/anonymize-catalog}
:reports {:child-fields [:metrics :logs :resource_events]
:query->wire-fn reports/reports-query->wire-v5
:query->wire-fn reports/reports-query->wire-v6
:anonymize-fn anon/anonymize-report}
:factsets {:child-fields [:facts]
:query->wire-fn factsets/factsets-query->wire-v4
......
......@@ -40,7 +40,7 @@
(do (log/infof "Importing report from archive entry '%s'" path)
(command-fn :store-report
(:store_report command-versions)
(reports/sanitize-report (utils/read-json-content tar-reader true))))
(utils/read-json-content tar-reader true)))
(file-pattern "facts")
(do (log/infof "Importing facts from archive entry '%s'" path)
(command-fn :replace-facts
......
......@@ -53,8 +53,8 @@
:resources [resource-wireformat-schema]
:noop (s/maybe s/Bool)
:transaction_uuid (s/maybe s/Str)
:metrics (s/maybe [metric-wireformat-schema])
:logs (s/maybe [log-wireformat-schema])
:metrics [metric-wireformat-schema]
:logs [log-wireformat-schema]
:environment s/Str
:status (s/maybe s/Str)})
......@@ -153,34 +153,40 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Reports Query -> Wire format conversions
(defn remove-reports-metadata [resource-events]
(map #(dissoc %
:report :certname :containing_class :configuration_version
:run_start_time :run_end_time :report_receive_time :environment)
resource-events))
(pls/defn-validated resource-events-query->wire-v5 :- [resource-event-v5-wireformat-schema]
[events :- resource-events-expanded-query-schema]
(sort-by
#(mapv % [:timestamp :resource_type :resource_title :property])
(map
#(dissoc % :report :certname :containing_class :configuration_version
:run_start_time :run_end_time :report_receive_time :environment)
(:data events))))
(pls/defn-validated logs-query->wire-v5 :- [log-wireformat-schema]
[logs :- logs-expanded-query-schema]
(:data logs))
(pls/defn-validated metrics-query->wire-v5 :- [metric-wireformat-schema]
[metrics :- metrics-expanded-query-schema]
(:data metrics))
[resource-events :- resource-events-expanded-query-schema]
(->> resource-events
:data
remove-reports-metadata
(sort-by #(mapv % [:timestamp :resource_type :resource_title :property]))))
(pls/defn-validated report-query->wire-v5 :- report-v5-wireformat-schema
[report :- report-query-schema]
(-> report
(dissoc :hash :receive_time :resources)
(update :resource_events resource-events-query->wire-v5)
(update :metrics metrics-query->wire-v5)
(update :logs logs-query->wire-v5)))
(update :metrics :data)
(update :logs :data)))
(defn reports-query->wire-v5 [reports]
(map report-query->wire-v5 reports))
(declare wire-v5->wire-v6)
(pls/defn-validated report-query->wire-v6 :- report-wireformat-schema
[report :- report-query-schema]
(-> report
report-query->wire-v5
wire-v5->wire-v6))
(defn reports-query->wire-v6 [reports]
(map report-query->wire-v6 reports))
(defn dash->underscore-report-keys [v5-report-or-older]
(->> v5-report-or-older
utils/dash->underscore-keys
......
......@@ -265,7 +265,7 @@
(is (coll? (anonymize-edges [test-edge] {} {})))
(is (= 1 (count (anonymize-edges [test-edge] {} {})))))))
(deftest test-anonymize-resource
(deftest test-anonymize-catalog-resource
(testing "should handle a resource"
(let [test-resource {"parameters" {"ensure" "present"}
"exported" true
......@@ -274,7 +274,7 @@
"tags" ["package"]
"title" "foo"
"type" "Package"}
result (anonymize-resource test-resource {} {})]
result (anonymize-catalog-resource test-resource {} {})]
(is (map? result))
(is (= #{"parameters" "exported" "line" "title" "tags" "type" "file"} (ks/keyset result)))))
(testing "should handle nil for file and line"
......@@ -283,11 +283,11 @@
"tags" ["package"]
"title" "foo"
"type" "Package"}
result (anonymize-resource test-resource {} {})]
result (anonymize-catalog-resource test-resource {} {})]
(is (map? result))
(is (= #{"parameters" "exported" "title" "tags" "type"} (ks/keyset result))))))
(deftest test-anonymize-resources
(deftest test-anonymize-catalog-resources
(testing "should handle a resource"
(let [test-resource {"parameters" {"ensure" "present"}
"exported" true
......@@ -296,7 +296,7 @@
"tags" ["package"]
"title" "foo"
"type" "Package"}
result (first (anonymize-resources [test-resource] {} {}))]
result (first (anonymize-catalog-resources [test-resource] {} {}))]
(is (= (ks/keyset test-resource)
(ks/keyset result)))
......@@ -310,52 +310,48 @@
"type"
"file"))))
(deftest test-anonymize-resource-event
(testing "should handle a resource event"
(let [test-event {"status" "noop"
"timestamp" "2013-03-04T19:56:34.000Z"
"resource_title" "foo"
"property" "ensure"
"message" "Ensure was absent now present"
"new_value" "present"
"old_value" "absent"
"resource_type" "Package"
"file" "/home/user/site.pp"
"line" 1
"containment_path" ["Stage[main]" "Foo" "Notify[hi]"]}
anonymized-event (anonymize-resource-event test-event {} {})]
(is (map? anonymized-event))
(is (= (keys test-event) (keys anonymized-event)))))
(testing "should handle a resource event with optionals"
(deftest test-anonymize-event
(testing "should handle a event"
(let [test-event {"status" "noop"
"timestamp" "2013-03-04T19:56:34.000Z"
"resource_title" "foo"
"property" "ensure"
"message" "Ensure was absent now present"
"new_value" "present"
"old_value" "absent"
"resource_type" "Package"
"file" nil
"line" nil
"containment_path" nil}
anonymized-event (anonymize-resource-event test-event {} {})]
"old_value" "absent"}
anonymized-event (anonymize-event test-event {} {})]
(is (map? anonymized-event))
(is (= (keys test-event) (keys anonymized-event))))))
(deftest test-anonymize-resource-events
(deftest test-anonymize-report-resource
(testing "should handle a resource event"
(let [test-event {"status" "noop"
"timestamp" "2013-03-04T19:56:34.000Z"
"resource_title" "foo"
"property" "ensure"
"message" "Ensure was absent now present"
"new_value" "present"
"old_value" "absent"
"resource_type" "Package"
"file" nil
"line" nil
"containment_path" nil}]
(is (coll? (anonymize-resource-events [test-event] {} {}))))))
(let [test-resource {"events" [{"status" "noop"
"timestamp" "2013-03-04T19:56:34.000Z"
"property" "ensure"
"message" "Ensure was absent now present"
"new_value" "present"
"old_value" "absent"}]
"timestamp" "2013-03-04T19:56:34.000Z"
"resource_title" "foo"
"resource_type" "Package"
"skipped" false
"file" "/home/user/site.pp"
"line" 1
"containment_path" ["Stage[main]" "Foo" "Notify[hi]"]}
anonymized-resource (anonymize-report-resource test-resource {} {})]
(is (map? anonymized-resource))
(is (= (keys test-resource) (keys anonymized-resource)))))
(testing "should handle a resource event with optionals"
(let [test-resource {"timestamp" "2013-03-04T19:56:34.000Z"
"resource_title" "foo"
"events" []
"resource_type" "Package"
"skipped" true
"file" nil
"line" nil
"containment_path" nil}
anonymized-resource (anonymize-report-resource test-resource {} {})]
(is (map? anonymized-resource))
(is (= (keys test-resource) (keys anonymized-resource))))))
(deftest test-anonymize-catalog
(testing "should accept basic valid data"
......
......@@ -28,29 +28,6 @@
RuntimeException #":timestamp \(not \(datetime\? \"foo\"\)\)"
(s/validate report-wireformat-schema (assoc-in report [:resources 0 :timestamp] "foo")))))))
(deftest test-sanitize-v5-resource-events
(testing "ensure extraneous keys are removed"
(let [test-data {:containment_path
["Stage[main]"
"My_pg"
"My_pg::Extension[puppetdb:pg_stat_statements]"
"Postgresql_psql[create extension pg_stat_statements on puppetdb]"]
:new_value "CREATE EXTENSION pg_stat_statements"
:message
"command changed '' to 'CREATE EXTENSION pg_stat_statements'"
:old_value nil
:status "success"
:line 16
:property "command"
:timestamp "2014-01-09T17:52:56.795Z"
:resource_type "Postgresql_psql"
:resource_title "create extension pg_stat_statements on puppetdb"
:file "/etc/puppet/modules/my_pg/manifests/extension.pp"
:extradata "foo"}
santized (sanitize-v5-resource-events [test-data])
expected [(dissoc test-data "extradata")]]
(= santized expected))))
(deftest test-sanitize-report
(testing "no action on valid reports"
(let [test-data (:basic reports)]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment