Commit 1ccd9486 authored by Molly Waggett's avatar Molly Waggett

(PE-15612) Add "producer" to PuppetDB reports

This commit migrates db to include "producer" column in reports table,
creates new report wireformat to include this field (v8), and adds it
as a queryable field.
parent d770b920
......@@ -106,7 +106,7 @@
(def supported-command-versions
{"replace facts" (version-range 2 4)
"replace catalog" (version-range 4 8)
"store report" (version-range 3 7)
"store report" (version-range 3 8)
"deactivate node" (version-range 1 3)})
(defn- die-on-header-payload-mismatch
......@@ -335,14 +335,14 @@
(defn store-report [{:keys [payload version annotations] :as command} db]
(let [{received-timestamp :received} annotations
latest-version-of-payload (case version
3 (report/wire-v3->wire-v7 payload received-timestamp)
4 (report/wire-v4->wire-v7 payload received-timestamp)
5 (report/wire-v5->wire-v7 payload)
6 (report/wire-v6->wire-v7 payload)
payload)
validated-payload (upon-error-throw-fatality
(s/validate report/report-wireformat-schema latest-version-of-payload))]
(s/validate report/report-wireformat-schema (case version
3 (report/wire-v3->wire-v8 payload received-timestamp)
4 (report/wire-v4->wire-v8 payload received-timestamp)
5 (report/wire-v5->wire-v8 payload)
6 (report/wire-v6->wire-v8 payload)
7 (report/wire-v7->wire-v8 payload)
payload)))]
(-> command
(assoc :payload validated-payload)
(store-report* db))))
......
......@@ -26,7 +26,7 @@
;; version of the `catalog` endpoint... or even to query what the latest
;; version of a command is. We should improve that.
{:replace_catalog 7
:store_report 6
:store_report 8
:replace_facts 4})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
......@@ -61,7 +61,7 @@
{"catalogs" {:query->wire-fn catalogs/catalogs-query->wire-v8
:anonymize-fn anon/anonymize-catalog
:json-encoded-fields [:edges :resources]}
"reports" {:query->wire-fn reports/reports-query->wire-v7
"reports" {:query->wire-fn reports/reports-query->wire-v8
:anonymize-fn anon/anonymize-report
:json-encoded-fields [:metrics :logs :resource_events :resources]}
"factsets" {:query->wire-fn factsets/factsets-query->wire-v4
......
......@@ -462,6 +462,9 @@
"producer_timestamp" {:type :timestamp
:queryable? true
:field :reports.producer_timestamp}
"producer" {:type :string
:queryable? true
:field :reports.producer}
"metrics" {:type :json
:queryable? false
:field {:select [(h/row-to-json :t)]
......
......@@ -50,6 +50,7 @@
:start_time pls/Timestamp
:end_time pls/Timestamp
:producer_timestamp pls/Timestamp
:producer (s/maybe s/Str)
:resources [resource-wireformat-schema]
:noop (s/maybe s/Bool)
:transaction_uuid (s/maybe s/Str)
......@@ -61,8 +62,12 @@
:environment s/Str
:status (s/maybe s/Str)})
(def report-v7-wireformat-schema
(-> report-wireformat-schema
(dissoc :producer)))
(def report-v6-wireformat-schema
(-> report-wireformat-schema
(-> report-v7-wireformat-schema
(dissoc :catalog_uuid :cached_catalog_status :code_id)))
(def resource-event-v5-wireformat-schema
......@@ -133,6 +138,7 @@
(s/optional-key :start_time) pls/Timestamp
(s/optional-key :end_time) pls/Timestamp
(s/optional-key :producer_timestamp) pls/Timestamp
(s/optional-key :producer) (s/maybe s/Str)
(s/optional-key :noop) (s/maybe s/Bool)
(s/optional-key :report_format) s/Int
(s/optional-key :configuration_version) s/Str
......@@ -188,21 +194,27 @@
:let [events (mapv #(dissoc % :file :line :resource_type :resource_title :containment_path) resource-events)]]
(assoc resource :events events))))
(defn wire-v6->wire-v7
(defn wire-v7->wire-v8
[{:keys [transaction_uuid] :as report}]
(utils/assoc-when report
:producer nil))
(defn wire-v6->wire-v8
[{:keys [transaction_uuid] :as report}]
(utils/assoc-when report
:catalog_uuid transaction_uuid
:cached_catalog_status nil
:code_id nil))
(-> report
(utils/assoc-when :catalog_uuid transaction_uuid
:cached_catalog_status nil
:code_id nil)
wire-v7->wire-v8))
(defn wire-v5->wire-v7
(defn wire-v5->wire-v8
[report]
(-> report
(update :resource_events resource-events-v5->resources)
(clojure.set/rename-keys {:resource_events :resources})
wire-v6->wire-v7))
wire-v6->wire-v8))
(defn wire-v4->wire-v7
(defn wire-v4->wire-v8
[report received-time]
(-> report
dash->underscore-report-keys
......@@ -210,23 +222,23 @@
:logs nil
:noop nil
:producer_timestamp received-time)
wire-v5->wire-v7))
wire-v5->wire-v8))
(defn wire-v3->wire-v7
(defn wire-v3->wire-v8
[report received-time]
(-> report
(assoc :status nil)
(wire-v4->wire-v7 received-time)))
(wire-v4->wire-v8 received-time)))
(pls/defn-validated report-query->wire-v7 :- report-wireformat-schema
(pls/defn-validated report-query->wire-v8 :- report-wireformat-schema
[report :- report-query-schema]
(-> report
report-query->wire-v5
wire-v5->wire-v7))
wire-v5->wire-v8))
(defn reports-query->wire-v7 [reports]
(map report-query->wire-v7 reports))
(defn reports-query->wire-v8 [reports]
(map report-query->wire-v8 reports))
(defn- resource->skipped-resource-events
"Fabricate a skipped resource-event"
......
......@@ -1048,6 +1048,11 @@
(jdbc/do-commands
"CREATE INDEX idx_certnames_latest_report_id on certnames(latest_report_id)"))
(defn add-producer-to-reports-catalogs-and-factsets
[]
(jdbc/do-commands
"ALTER TABLE reports ADD COLUMN producer TEXT"))
(def migrations
"The available migrations, as a map from migration version to migration function."
{28 init-through-2-3-8
......@@ -1070,7 +1075,8 @@
42 add-support-for-historical-catalogs
43 add-indexes-for-reports-summary-query
44 add-catalog-uuid-to-reports-and-catalogs
45 index-certnames-latest-report-id})
45 index-certnames-latest-report-id
46 add-producer-to-reports-catalogs-and-factsets})
(def desired-schema-version (apply max (keys migrations)))
......
......@@ -1304,7 +1304,7 @@
received-timestamp :- pls/Timestamp
update-latest-report? :- s/Bool]
(time! (:store-report performance-metrics)
(let [{:keys [puppet_version certname report_format configuration_version
(let [{:keys [puppet_version certname report_format configuration_version producer
producer_timestamp start_time end_time transaction_uuid environment
status noop metrics logs resources resource_events catalog_uuid
code_id cached_catalog_status]
......@@ -1325,6 +1325,7 @@
:certname certname
:report_format report_format
:configuration_version configuration_version
:producer producer
:producer_timestamp producer_timestamp
:start_time start_time
:end_time end_time
......
......@@ -25,7 +25,7 @@
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace catalog" 7 example-catalog)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"store report" 6 example-report)
"store report" 8 example-report)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace facts" 4 example-facts)
......
......@@ -34,7 +34,7 @@
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace catalog" 8 example-catalog)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"store report" 7 example-report)
"store report" 8 example-report)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace facts" 4 example-facts)
......@@ -78,7 +78,7 @@
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace catalog" 8 example-catalog)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"store report" 7 example-report)
"store report" 8 example-report)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace facts" 4 example-facts)
......@@ -116,7 +116,7 @@
example-catalog2)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"store report" 7 example-report)
"store report" 8 example-report)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace facts" 4 example-facts)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) "bar.com"
......
......@@ -26,7 +26,7 @@
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace catalog" 7 example-catalog)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"store report" 6 example-report)
"store report" 8 example-report)
(svc-utils/sync-command-post (svc-utils/pdb-cmd-url) example-certname
"replace facts" 4 example-facts)
......
......@@ -1247,9 +1247,20 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def v8-report
(-> (:basic report-examples/reports)
reports/report-query->wire-v8))
(def v7-report
(-> v8-report
(dissoc :producer)))
(def v6-report
(-> v7-report
(dissoc :catalog_uuid :cached_catalog_status :code_id)))
(def v5-report
(-> (:basic report-examples/reports)
(assoc :environment "DEV")
reports/report-query->wire-v5))
(def v4-report
......@@ -1259,11 +1270,35 @@
(def store-report-name (command-names :store-report))
(deftest store-v8-report-test
(let [command {:command store-report-name
:version 8
:payload v8-report}]
(with-test-db
(test-msg-handler command publish discard-dir
(is (= [(select-keys v8-report [:certname :producer])]
(-> (str "select certname, producer"
" from reports")
query-to-vec)))
(is (= 0 (times-called publish)))
(is (empty? (fs/list-dir discard-dir)))))))
(deftest store-v7-report-test
(let [command {:command store-report-name
:version 7
:payload v7-report}]
(with-test-db
(test-msg-handler command publish discard-dir
(is (= [(select-keys v7-report [:certname :catalog_uuid :cached_catalog_status :code_id])]
(->> (str "select certname, catalog_uuid, cached_catalog_status, code_id"
" from reports")
query-to-vec
(map (fn [row] (update row :catalog_uuid sutils/parse-db-uuid))))))
(is (= 0 (times-called publish)))
(is (empty? (fs/list-dir discard-dir)))))))
(deftest store-v6-report-test
(let [v6-report (-> v5-report
(update :resource_events reports/resource-events-v5->resources)
(clojure.set/rename-keys {:resource_events :resources}))
command {:command store-report-name
(let [command {:command store-report-name
:version 6
:payload v6-report}]
(with-test-db
......
......@@ -10,6 +10,7 @@
:start_time "2011-01-01T12:00:00-03:00"
:end_time "2011-01-01T12:10:00-03:00"
:producer_timestamp "2011-01-01T12:11:00-03:00"
:producer "bar.com"
:catalog_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:code_id nil
:cached_catalog_status "not_used"
......@@ -103,6 +104,7 @@
:start_time "2013-08-28T19:00:00-03:00"
:end_time "2013-08-28T19:10:00-03:00"
:producer_timestamp "2013-08-28T19:11:00-03:00"
:producer "bar.com"
:environment "DEV"
:status "unchanged"
:noop true
......@@ -187,6 +189,7 @@
:start_time "2011-01-03T12:00:00-03:00"
:end_time "2011-01-03T12:10:00-03:00"
:producer_timestamp "2011-01-03T12:11:00-03:00"
:producer "bar.com"
:catalog_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:code_id nil
:cached_catalog_status "not_used"
......@@ -277,6 +280,7 @@
:start_time "2011-01-03T12:00:00-03:00"
:end_time "2011-01-03T12:10:00-03:00"
:producer_timestamp "2011-01-03T12:11:00-03:00"
:producer "bar.com"
:environment "DEV"
:status "unchanged"
:noop false
......
......@@ -40,7 +40,7 @@
(doseq [field ["certname" "hash" "puppet_version" "report_format"
"configuration_version" "start_time" "end_time"
"transaction_uuid" "status"]
"transaction_uuid" "status" "producer"]
:let [field-kwd (keyword field)]]
(testing (format "should return all reports for a %s" field)
(let [result (query-response method endpoint ["=" field (get basic-with-hash field-kwd)])
......
......@@ -9,11 +9,11 @@
[schema.core :as s]))
(let [report (-> (:basic reports)
report-query->wire-v7)]
report-query->wire-v8)]
(deftest test-validate
(testing "should accept a valid v7 report"
(testing "should accept a valid v8 report"
(is (= report (s/validate report-wireformat-schema report))))
(testing "should fail when a report is missing a key"
......@@ -32,10 +32,23 @@
(sp/transform [:resource-events sp/ALL sp/ALL]
#(update % 0 utils/underscores->dashes))))
(def v8-example-report
(-> reports
:basic
report-query->wire-v8))
(def v7-example-report
(-> v8-example-report
(dissoc :producer)))
(def v6-example-report
(-> v7-example-report
(dissoc :code_id :catalog_uuid :cached_catalog_status)))
(def v5-example-report
(-> reports
:basic
(dissoc :code_id :catalog_uuid :cached_catalog_status)
(dissoc :code_id :catalog_uuid :cached_catalog_status :producer)
report-query->wire-v5))
(def v4-example-report
......@@ -43,26 +56,40 @@
underscore->dash-report-keys
(dissoc :logs :metrics :noop :producer-timestamp)))
(deftest test-v7-conversion
(let [v7-report v7-example-report
v8-report (wire-v7->wire-v8 v7-report)]
(is (s/validate report-v7-wireformat-schema v7-report))
(is (s/validate report-wireformat-schema v8-report))))
(deftest test-v6-conversion
(let [v6-report v6-example-report
v8-report (wire-v6->wire-v8 v6-report)]
(is (s/validate report-v6-wireformat-schema v6-report))
(is (s/validate report-wireformat-schema v8-report))))
(deftest test-v5-conversion
(let [v5-report v5-example-report
v7-report (wire-v5->wire-v7 v5-report)]
v8-report (wire-v5->wire-v8 v5-report)]
(is (s/validate report-v5-wireformat-schema v5-report))
(is (s/validate report-wireformat-schema v7-report))))
(is (s/validate report-wireformat-schema v8-report))))
(deftest test-v4-conversion
(let [current-time (now)
v7-report (wire-v4->wire-v7 v4-example-report current-time)]
v8-report (wire-v4->wire-v8 v4-example-report current-time)]
(is (s/validate report-v4-wireformat-schema v4-example-report))
(is (s/validate report-wireformat-schema v7-report))
(is (= current-time (:producer_timestamp v7-report)))))
(is (s/validate report-wireformat-schema v8-report))
(is (= current-time (:producer_timestamp v8-report)))))
(deftest test-v3-conversion
(let [current-time (now)
v3-report (dissoc v4-example-report :status)
v7-report (wire-v3->wire-v7 v3-report current-time)]
v8-report (wire-v3->wire-v8 v3-report current-time)]
(is (s/validate report-v3-wireformat-schema v3-report))
(is (s/validate report-wireformat-schema (wire-v3->wire-v7 v3-report current-time)))
(is (= current-time (:producer_timestamp v7-report)))
(is (nil? (:status v7-report)))))
(is (s/validate report-wireformat-schema (wire-v3->wire-v8 v3-report current-time)))
(is (= current-time (:producer_timestamp v8-report)))
(is (nil? (:status v8-report)))))
......@@ -206,7 +206,7 @@
(deftest report-dedupe
(let [report-query->hash (comp report-identity-hash
normalize-report
reports/report-query->wire-v7)
reports/report-query->wire-v8)
report (:basic reports)
report2-events (get-in reports [:basic4 :resource_events :data])
report2 (assoc-in report [:resource_events :data] report2-events)
......
......@@ -460,6 +460,21 @@
(map #(dissoc % :same))
set))))))
(deftest test-add-producer-to-reports-catalogs-and-factsets-migration
(clear-db-for-testing!)
(fast-forward-to-migration! 45)
(let [before-migration (schema-info-map *db*)]
(apply-migration-for-testing! 46)
(let [schema-diff (diff-schema-maps before-migration (schema-info-map *db*))]
(is (= (set [{:same nil :left-only nil
:right-only {:numeric_scale nil :column_default nil
:character_octet_length 1073741824 :datetime_precision nil
:nullable? "YES" :character_maximum_length nil
:numeric_precision nil :numeric_precision_radix nil
:data_type "text" :column_name "producer"
:table_name "reports"}}])
(set (:table-diff schema-diff)))))))
(deftest test-migrate-from-unsupported-version
(clear-db-for-testing!)
(fast-forward-to-migration! 28)
......
......@@ -1404,7 +1404,7 @@
(let [timestamp (now)
{:keys [certname] :as report} (:basic reports)
report-hash (-> report
report/report-query->wire-v7
report/report-query->wire-v8
normalize-report
shash/report-identity-hash)]
......@@ -1443,6 +1443,13 @@
[{:certname (:certname report)
:environment_id (environment-id "DEV")}])))
(deftest-db report-storage-with-producer
(store-example-report! (assoc report :producer "bar.com") timestamp)
(is (= (query-to-vec ["SELECT certname, producer FROM reports"])
[{:certname (:certname report)
:producer "bar.com"}])))
(deftest-db report-storage-with-status
(is (nil? (status-id "unchanged")))
......
......@@ -29,7 +29,7 @@
(-> (svc-utils/pdb-query-url)
(svc-utils/get-reports certname)
tur/munge-reports
reports/reports-query->wire-v7
reports/reports-query->wire-v8
vec))
(defn get-factsets [certname]
......@@ -64,7 +64,7 @@
:basic
(assoc :certname example-certname)
tur/munge-report
reports/report-query->wire-v7))
reports/report-query->wire-v8))
(defn munge-tar-map
[tar-map]
......
......@@ -21,7 +21,7 @@
(-> (:basic reports)
(change-certname node-name)
(assoc :end_time (now))
report/report-query->wire-v7))
report/report-query->wire-v8))
(defn store-example-nodes
[]
......
......@@ -44,7 +44,7 @@
([example-report timestamp]
(store-example-report! example-report timestamp true))
([example-report timestamp update-latest-report?]
(let [example-report (reports/report-query->wire-v7 example-report)
(let [example-report (reports/report-query->wire-v8 example-report)
report-hash (shash/report-identity-hash
(scf-store/normalize-report example-report))]
(scf-store/maybe-activate-node! (:certname example-report) timestamp)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment