Commit 797ca49d authored by Andrew Roetker's avatar Andrew Roetker

(PDB-2126) Bump command versions for catalogs and reports

This commit adds new command versions (wire-formats) for reports and
catalogs associated with the new catalog_uuid, cached_catalog_reason
and code_id fields.
parent 9e7ca9af
......@@ -98,7 +98,7 @@
(def ^:const catalog-version
"Constant representing the version number of the PuppetDB
catalog format"
7)
8)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Catalog Schemas
......@@ -115,6 +115,7 @@
:version s/Str
:environment (s/maybe s/Str)
:transaction_uuid (s/maybe s/Str)
:catalog_uuid (s/maybe s/Str)
:producer_timestamp pls/Timestamp
:code_id (s/maybe s/Str)
......@@ -133,8 +134,11 @@
:resources (s/cond-pre [{s/Any s/Any}]
{s/Any {s/Any s/Any}})})
(def catalog-v7-wireformat-schema
(dissoc catalog-wireformat-schema :catalog_uuid))
(def catalog-v6-wireformat-schema
(dissoc catalog-wireformat-schema :code_id))
(dissoc catalog-v7-wireformat-schema :code_id))
(def edge-query-schema
"Schema for validating a single edge."
......@@ -179,6 +183,7 @@
(s/optional-key :producer_timestamp) (s/maybe pls/Timestamp)
(s/optional-key :resources) resources-expanded-query-schema
(s/optional-key :transaction_uuid) (s/maybe s/Str)
(s/optional-key :catalog_uuid) (s/maybe s/Str)
(s/optional-key :code_id) (s/maybe s/Str)
(s/optional-key :version) s/Str})
......@@ -193,6 +198,9 @@
:transaction_uuid nil
:environment nil))
(defn wire-v7->wire-v8 [{:keys [transaction_uuid] :as catalog}]
(assoc catalog :catalog_uuid transaction_uuid))
(defn wire-v6->wire-v7 [catalog]
(assoc catalog :code_id nil))
......@@ -232,7 +240,7 @@
[edge]
{:pre [(:relationship edge)]
:post [(keyword? (:relationship %))]}
(update-in edge [:relationship] keyword))
(update edge :relationship keyword))
(defn transform-edges
"Transforms every edge of the given `catalog` and converts the edges into a set."
......@@ -394,6 +402,13 @@
7 nil))
(defmethod parse-catalog 7
[catalog version _]
{:pre [(map? catalog)]
:post [(map? %)]}
(parse-catalog (wire-v7->wire-v8 catalog)
8 nil))
(defmethod parse-catalog 8
[catalog version _]
{:pre [(map? catalog)]
:post [(map? %)]}
......@@ -410,7 +425,7 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Catalog Query -> Wire format conversions
(pls/defn-validated edge-query->wire-v7
(pls/defn-validated edge-query->wire-v8
[edge :- edge-query-schema]
{:source
{:title (:source_title edge)
......@@ -420,28 +435,28 @@
:type (:target_type edge)}
:relationship (:relationship edge)})
(pls/defn-validated edges-expanded->wire-v7
(pls/defn-validated edges-expanded->wire-v8
[edges :- edges-expanded-query-schema]
(map edge-query->wire-v7
(map edge-query->wire-v8
(:data edges)))
(pls/defn-validated resource-query->wire-v7
(pls/defn-validated resource-query->wire-v8
[resource :- resource-query-schema]
(-> resource
(dissoc :resource :certname :environment)
(kitchensink/dissoc-if-nil :file :line)))
(pls/defn-validated resources-expanded->wire-v7
(pls/defn-validated resources-expanded->wire-v8
[resources :- resources-expanded-query-schema]
(map resource-query->wire-v7
(map resource-query->wire-v8
(:data resources)))
(pls/defn-validated catalog-query->wire-v7 :- catalog-wireformat-schema
(pls/defn-validated catalog-query->wire-v8 :- catalog-wireformat-schema
[catalog :- catalog-query-schema]
(-> catalog
(dissoc :hash)
(update :edges edges-expanded->wire-v7)
(update :resources resources-expanded->wire-v7)))
(update :edges edges-expanded->wire-v8)
(update :resources resources-expanded->wire-v8)))
(defn catalogs-query->wire-v7 [catalogs]
(map catalog-query->wire-v7 catalogs))
(defn catalogs-query->wire-v8 [catalogs]
(map catalog-query->wire-v8 catalogs))
......@@ -97,8 +97,8 @@
(def supported-command-versions
{"replace facts" (version-range 2 4)
"replace catalog" (version-range 4 7)
"store report" (version-range 3 6)
"replace catalog" (version-range 4 8)
"store report" (version-range 3 7)
"deactivate node" (version-range 1 3)})
(defn- die-on-header-payload-mismatch
......@@ -280,7 +280,6 @@
(assoc :payload validated-payload)
(replace-facts* db))))
;; Node deactivation
(defn deactivate-node-wire-v2->wire-3 [deactive-node]
......@@ -328,9 +327,10 @@
(defn store-report [{:keys [payload version annotations] :as command} db]
(let [{received-timestamp :received} annotations
latest-version-of-payload (case version
3 (report/wire-v3->wire-v6 payload received-timestamp)
4 (report/wire-v4->wire-v6 payload received-timestamp)
5 (report/wire-v5->wire-v6 payload)
3 (report/wire-v3->wire-v7 payload received-timestamp)
4 (report/wire-v4->wire-v7 payload received-timestamp)
5 (report/wire-v5->wire-v7 payload)
6 (report/wire-v6->wire-v7 payload)
payload)
validated-payload (upon-error-throw-fatality
(s/validate report/report-wireformat-schema latest-version-of-payload))]
......
......@@ -57,10 +57,10 @@
(map #(export-datum->tar-item entity %) data))
(def export-info
{"catalogs" {:query->wire-fn catalogs/catalogs-query->wire-v7
{"catalogs" {:query->wire-fn catalogs/catalogs-query->wire-v8
:anonymize-fn anon/anonymize-catalog
:json-encoded-fields [:edges :resources]}
"reports" {:query->wire-fn reports/reports-query->wire-v6
"reports" {:query->wire-fn reports/reports-query->wire-v7
:anonymize-fn anon/anonymize-report
:json-encoded-fields [:metrics :logs :resource_events :resources]}
"factsets" {:query->wire-fn factsets/factsets-query->wire-v4
......
......@@ -426,9 +426,18 @@
"transaction_uuid" {:type :string
:queryable? true
:field (hsql-uuid-as-str :reports.transaction_uuid)}
"catalog_uuid" {:type :string
:queryable? true
:field (hsql-uuid-as-str :reports.catalog_uuid)}
"noop" {:type :boolean
:queryable? true
:field :reports.noop}
"code_id" {:type :string
:queryable? true
:field :reports.code_id}
"cached_catalog_reason" {:type :string
:queryable? true
:field :reports.cached_catalog_reason}
"environment" {:type :string
:queryable? true
:field :environments.environment}
......@@ -497,6 +506,9 @@
"transaction_uuid" {:type :string
:queryable? true
:field (hsql-uuid-as-str :c.transaction_uuid)}
"catalog_uuid" {:type :string
:queryable? true
:field (hsql-uuid-as-str :c.catalog_uuid)}
"code_id" {:type :string
:queryable? true
:field :c.code_id}
......
......@@ -53,18 +53,25 @@
:resources [resource-wireformat-schema]
:noop (s/maybe s/Bool)
:transaction_uuid (s/maybe s/Str)
:catalog_uuid (s/maybe s/Str)
:code_id (s/maybe s/Str)
:cached_catalog_reason (s/maybe s/Str)
:metrics [metric-wireformat-schema]
:logs [log-wireformat-schema]
:environment s/Str
:status (s/maybe s/Str)})
(def report-v6-wireformat-schema
(-> report-wireformat-schema
(dissoc :catalog_uuid :cached_catalog_reason :code_id)))
(def resource-event-v5-wireformat-schema
(-> resource-wireformat-schema
(dissoc :skipped :events)
(merge event-wireformat-schema)))
(def report-v5-wireformat-schema
(-> report-wireformat-schema
(-> report-v6-wireformat-schema
(dissoc :resources)
(assoc :resource_events [resource-event-v5-wireformat-schema])))
......@@ -133,7 +140,10 @@
(s/optional-key :metrics) metrics-expanded-query-schema
(s/optional-key :logs) logs-expanded-query-schema
(s/optional-key :resource_events) resource-events-expanded-query-schema
(s/optional-key :transaction_uuid) s/Str
(s/optional-key :transaction_uuid) (s/maybe s/Str)
(s/optional-key :catalog_uuid) (s/maybe s/Str)
(s/optional-key :code_id) (s/maybe s/Str)
(s/optional-key :cached_catalog_reason) (s/maybe s/Str)
(s/optional-key :status) (s/maybe s/Str)})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
......@@ -147,15 +157,16 @@
:report :certname :containing_class :configuration_version
:run_start_time :run_end_time :report_receive_time :environment))))
(pls/defn-validated report-query->wire-v5 :- report-v5-wireformat-schema
[report :- report-query-schema]
(defn report-query->wire-v5
[report]
(-> report
(dissoc :hash :receive_time :resources)
(update :resource_events resource-events-query->wire-v5)
(update :metrics :data)
(update :logs :data)))
(defn reports-query->wire-v5 [reports]
(pls/defn-validated reports-query->wire-v5 :- [report-v5-wireformat-schema]
[reports :- [report-query-schema]]
(map report-query->wire-v5 reports))
(defn dash->underscore-report-keys [v5-report-or-older]
......@@ -177,13 +188,21 @@
:let [events (mapv #(dissoc % :file :line :resource_type :resource_title :containment_path) resource-events)]]
(assoc resource :events events))))
(defn wire-v5->wire-v6
(defn wire-v6->wire-v7
[{:keys [transaction_uuid] :as report}]
(utils/assoc-when report
:catalog_uuid transaction_uuid
:cached_catalog_reason nil
:code_id nil))
(defn wire-v5->wire-v7
[report]
(-> report
(update :resource_events resource-events-v5->resources)
(clojure.set/rename-keys {:resource_events :resources})))
(clojure.set/rename-keys {:resource_events :resources})
wire-v6->wire-v7))
(defn wire-v4->wire-v6
(defn wire-v4->wire-v7
[report received-time]
(-> report
dash->underscore-report-keys
......@@ -191,23 +210,23 @@
:logs nil
:noop nil
:producer_timestamp received-time)
wire-v5->wire-v6))
wire-v5->wire-v7))
(defn wire-v3->wire-v6
(defn wire-v3->wire-v7
[report received-time]
(-> report
(assoc :status nil)
(wire-v4->wire-v6 received-time)))
(wire-v4->wire-v7 received-time)))
(pls/defn-validated report-query->wire-v6 :- report-wireformat-schema
(pls/defn-validated report-query->wire-v7 :- report-wireformat-schema
[report :- report-query-schema]
(-> report
report-query->wire-v5
wire-v5->wire-v6))
wire-v5->wire-v7))
(defn reports-query->wire-v6 [reports]
(map report-query->wire-v6 reports))
(defn reports-query->wire-v7 [reports]
(map report-query->wire-v7 reports))
(defn- resource->skipped-resource-events
"Fabricate a skipped resource-event"
......
......@@ -1016,6 +1016,14 @@
"ALTER TABLE resource_events ADD CONSTRAINT resource_events_report_id_fkey
FOREIGN KEY (report_id) REFERENCES reports(id) ON DELETE CASCADE"))
(defn add-catalog-uuid-to-reports-and-catalogs
[]
(jdbc/do-commands
"ALTER TABLE reports ADD COLUMN cached_catalog_reason TEXT"
"ALTER TABLE reports ADD COLUMN code_id TEXT"
"UPDATE catalogs SET catalog_uuid=catalogs.transaction_uuid WHERE hash is NULL"
"UPDATE reports SET catalog_uuid=reports.transaction_uuid WHERE hash is NULL"))
(def migrations
"The available migrations, as a map from migration version to migration function."
{28 init-through-2-3-8
......@@ -1036,7 +1044,8 @@
40 fix-bytea-expression-indexes-to-use-encode
41 factset-hash-field-not-nullable
42 add-support-for-historical-catalogs
43 add-indexes-for-reports-summary-query})
43 add-indexes-for-reports-summary-query
44 add-catalog-uuid-to-reports-and-catalogs})
(def desired-schema-version (apply max (keys migrations)))
......
......@@ -379,7 +379,14 @@
(pls/defn-validated catalog-row-map
"Creates a row map for the catalogs table, optionally adding envrionment when it was found"
[hash
{:keys [edges resources version code_id transaction_uuid environment producer_timestamp]} :- catalog-schema
{:keys [edges
resources
version
code_id
transaction_uuid
catalog_uuid
environment
producer_timestamp]} :- catalog-schema
received-timestamp :- pls/Timestamp]
(let [catalogs-jsonb? @store-catalogs-jsonb-columns?]
{:hash (sutils/munge-hash-for-storage hash)
......@@ -387,6 +394,7 @@
:resources (when catalogs-jsonb? (munge-resources-for-storage (vals resources)))
:catalog_version version
:transaction_uuid (sutils/munge-uuid-for-storage transaction_uuid)
:catalog_uuid (sutils/munge-uuid-for-storage catalog_uuid)
:timestamp (to-timestamp received-timestamp)
:code_id code_id
:environment_id (ensure-environment environment)
......@@ -1218,13 +1226,17 @@
(time! (:store-report performance-metrics)
(let [{:keys [puppet_version certname report_format configuration_version
producer_timestamp start_time end_time transaction_uuid environment
status noop metrics logs resources resource_events]
status noop metrics logs resources resource_events catalog_uuid
code_id cached_catalog_reason]
:as report} (normalize-report orig-report)
report-hash (shash/report-identity-hash report)]
(jdbc/with-db-transaction []
(let [certname-id (certname-id certname)
row-map {:hash (sutils/munge-hash-for-storage report-hash)
:transaction_uuid (sutils/munge-uuid-for-storage transaction_uuid)
:catalog_uuid (sutils/munge-uuid-for-storage catalog_uuid)
:code_id code_id
:cached_catalog_reason cached_catalog_reason
:metrics (sutils/munge-jsonb-for-storage metrics)
:logs (sutils/munge-jsonb-for-storage logs)
:resources (sutils/munge-jsonb-for-storage resources)
......
......@@ -44,5 +44,6 @@
} ],
"version" : "1330463446",
"transaction_uuid" : "68b08e2a-eeb1-4322-b241-bfdf151d294c",
"catalog_uuid" : "68b08e2a-eeb1-4322-b241-bfdf151d294c",
"environment" : "DEV"
}
......@@ -88,9 +88,11 @@
(testing "key validation"
(let [catalog (dissoc (:basic catalogs) :api_version)
v6-catalog (dissoc catalog :code_id)]
v7-catalog (dissoc catalog :catalog_uuid)
v6-catalog (dissoc v7-catalog :code_id)]
(testing "should accept catalogs with the correct set of keys"
(= catalog (s/validate catalog-wireformat-schema catalog))
(= v7-catalog (s/validate catalog-v7-wireformat-schema v7-catalog))
(= v6-catalog (s/validate catalog-v6-wireformat-schema v6-catalog)))
(testing "should fail if the catalog has an extra key"
......
......@@ -6,6 +6,7 @@
{:certname "empty.catalogs.com"
:version "1330463884"
:transaction_uuid "aaaaaaaa-1111-aaaa-1111-aaaaaaaaaaaa"
:catalog_uuid "aaaaaaaa-1111-aaaa-1111-aaaaaaaaaaaa"
:environment nil
:code_id nil
:producer_timestamp "2014-07-10T22:33:54.781Z"
......@@ -35,6 +36,7 @@
{:certname "basic.catalogs.com"
:code_id nil
:transaction_uuid "68b08e2a-eeb1-4322-b241-bfdf151d294b"
:catalog_uuid "68b08e2a-eeb1-4322-b241-bfdf151d294b"
:environment "DEV"
:version "123456789"
:producer_timestamp "2014-07-10T22:33:54.781Z"
......@@ -76,6 +78,7 @@
{:certname "invalid.catalogs.com"
:code_id nil
:transaction_uuid "68b08e2a-eeb1-4322-b241-bfdf151d294b"
:catalog_uuid "68b08e2a-eeb1-4322-b241-bfdf151d294b"
:version 123456789
:edges #{{:source {:type "Class" :title "foobar"}
:target {:type "File" :title "does not exist"}
......@@ -127,6 +130,7 @@
:type "Node"}]
:version "1332533763"
:transaction_uuid "68b08e2a-eeb1-4322-b241-bfdf151d294b"
:catalog_uuid "68b08e2a-eeb1-4322-b241-bfdf151d294b"
:environment "DEV"
:producer_timestamp "2014-07-10T22:33:54.781Z"})
......
......@@ -10,6 +10,9 @@
:start_time "2011-01-01T12:00:00-03:00"
:end_time "2011-01-01T12:10:00-03:00"
:producer_timestamp "2011-01-01T12:11:00-03:00"
:catalog_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:code_id nil
:cached_catalog_reason "not_used"
:environment "DEV"
:status "unchanged"
:noop false
......@@ -93,6 +96,9 @@
:puppet_version "3.0.1"
:report_format 4
:transaction_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:catalog_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:code_id nil
:cached_catalog_reason "not_used"
:configuration_version "bja3985a23"
:start_time "2013-08-28T19:00:00-03:00"
:end_time "2013-08-28T19:10:00-03:00"
......@@ -181,6 +187,9 @@
:start_time "2011-01-03T12:00:00-03:00"
:end_time "2011-01-03T12:10:00-03:00"
:producer_timestamp "2011-01-03T12:11:00-03:00"
:catalog_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:code_id nil
:cached_catalog_reason "not_used"
:environment "DEV"
:status "unchanged"
:noop false
......@@ -261,6 +270,9 @@
:puppet_version "3.0.1"
:report_format 4
:transaction_uuid "e1e561ba-212f-11e3-9d58-60a44c233a9d"
:catalog_uuid "5ea3a70b-84c8-426c-813c-dd6492fb829b"
:code_id nil
:cached_catalog_reason "not_used"
:configuration_version "a81jasj123"
:start_time "2011-01-03T12:00:00-03:00"
:end_time "2011-01-03T12:10:00-03:00"
......
......@@ -86,7 +86,7 @@
(is (= (count expected) (count response-body)))
(is (= (sort (map :certname expected)) (sort (map :certname response-body))))
(is (= (extract-tags expected)
(extract-tags (catalogs/catalogs-query->wire-v7 response-body)))))))
(extract-tags (catalogs/catalogs-query->wire-v8 response-body)))))))
(testing "projection queries"
(are [query expected]
......
......@@ -9,11 +9,11 @@
[schema.core :as s]))
(let [report (-> (:basic reports)
report-query->wire-v6)]
report-query->wire-v7)]
(deftest test-validate
(testing "should accept a valid v5 report"
(testing "should accept a valid v7 report"
(is (= report (s/validate report-wireformat-schema report))))
(testing "should fail when a report is missing a key"
......@@ -32,33 +32,37 @@
(sp/transform [:resource-events sp/ALL sp/ALL]
#(update % 0 utils/underscores->dashes))))
(def v4-example-report
(def v5-example-report
(-> reports
:basic
report-query->wire-v5
(dissoc :code_id :catalog_uuid :cached_catalog_reason)
report-query->wire-v5))
(def v4-example-report
(-> v5-example-report
underscore->dash-report-keys
(dissoc :logs :metrics :noop :producer-timestamp)))
(deftest test-v5-conversion
(let [v5-report (-> reports :basic report-query->wire-v5)
v6-report (wire-v5->wire-v6 v5-report)]
(let [v5-report v5-example-report
v7-report (wire-v5->wire-v7 v5-report)]
(is (s/validate report-v5-wireformat-schema v5-report))
(is (s/validate report-wireformat-schema v6-report))))
(is (s/validate report-wireformat-schema v7-report))))
(deftest test-v4-conversion
(let [current-time (now)
v6-report (wire-v4->wire-v6 v4-example-report current-time)]
v7-report (wire-v4->wire-v7 v4-example-report current-time)]
(is (s/validate report-v4-wireformat-schema v4-example-report))
(is (s/validate report-wireformat-schema v6-report))
(is (= current-time (:producer_timestamp v6-report)))))
(is (s/validate report-wireformat-schema v7-report))
(is (= current-time (:producer_timestamp v7-report)))))
(deftest test-v3-conversion
(let [current-time (now)
v3-report (dissoc v4-example-report :status)
v6-report (wire-v3->wire-v6 v3-report current-time)]
v7-report (wire-v3->wire-v7 v3-report current-time)]
(is (s/validate report-v3-wireformat-schema v3-report))
(is (s/validate report-wireformat-schema (wire-v3->wire-v6 v3-report current-time)))
(is (= current-time (:producer_timestamp v6-report)))
(is (nil? (:status v6-report)))))
(is (s/validate report-wireformat-schema (wire-v3->wire-v7 v3-report current-time)))
(is (= current-time (:producer_timestamp v7-report)))
(is (nil? (:status v7-report)))))
......@@ -206,7 +206,7 @@
(deftest report-dedupe
(let [report-query->hash (comp report-identity-hash
normalize-report
reports/report-query->wire-v6)
reports/report-query->wire-v7)
report (:basic reports)
report2-events (get-in reports [:basic4 :resource_events :data])
report2 (assoc-in report [:resource_events :data] report2-events)
......
......@@ -1302,7 +1302,7 @@
(let [timestamp (now)
{:keys [certname] :as report} (:basic reports)
report-hash (-> report
report/report-query->wire-v6
report/report-query->wire-v7
normalize-report
shash/report-identity-hash)]
......
......@@ -22,14 +22,14 @@
(defn get-catalogs [certname]
(-> (svc-utils/pdb-query-url)
(svc-utils/get-catalogs certname)
catalogs/catalogs-query->wire-v7
catalogs/catalogs-query->wire-v8
vec))
(defn get-reports [certname]
(-> (svc-utils/pdb-query-url)
(svc-utils/get-reports certname)
tur/munge-reports
reports/reports-query->wire-v6
reports/reports-query->wire-v7
vec))
(defn get-factsets [certname]
......@@ -60,7 +60,7 @@
:basic
(assoc :certname example-certname)
tur/munge-report
reports/report-query->wire-v6))
reports/report-query->wire-v7))
(defn munge-tar-map
[tar-map]
......
......@@ -21,7 +21,7 @@
(-> (:basic reports)
(change-certname node-name)
(assoc :end_time (now))
report/report-query->wire-v6))
report/report-query->wire-v7))
(defn store-example-nodes
[]
......
......@@ -44,7 +44,7 @@
([example-report timestamp]
(store-example-report! example-report timestamp true))
([example-report timestamp update-latest-report?]
(let [example-report (reports/report-query->wire-v6 example-report)
(let [example-report (reports/report-query->wire-v7 example-report)
report-hash (shash/report-identity-hash
(scf-store/normalize-report example-report))]
(scf-store/maybe-activate-node! (:certname example-report) timestamp)
......@@ -88,8 +88,9 @@
"Convert actual results for reports queries to wire format ready for comparison."
[reports]
(set
(map munge-report-for-comparison
(reports/reports-query->wire-v5 reports))))
(map (comp munge-report-for-comparison
reports/report-query->wire-v5)
reports)))
(defn enumerated-resource-events-map
[resource-events]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment