Commit 3ac87dbb authored by Russell Mull's avatar Russell Mull

(PDB-1487) Add producer_timestamp to store-report

The store-report command contains start_time and end_time, but those
are generated on the agent. This adds a producer_timestamp field to the
command, which is generated on the master. It can then be reliably
compared with the producer_timestamp on the deactivate-node command.
parent 32ee5f88
......@@ -56,9 +56,11 @@ The below fields are allowed as filter criteria and are returned in all response
* `configuration_version` (string): an identifier string that Puppet uses to match a specific catalog for a node to a specific Puppet run.
* `start_time` (timestamp): is the time at which the Puppet run began. Timestamps are always [ISO-8601][8601] compatible date/time strings.
* `start_time` (timestamp): is the time on the agent at which the Puppet run began. Timestamps are always [ISO-8601][8601] compatible date/time strings.
* `end_time` (timestamp): is the time at which the Puppet run ended. Timestamps are always [ISO-8601][8601] compatible date/time strings.
* `end_time` (timestamp): is the time on the agent at which the Puppet run ended. Timestamps are always [ISO-8601][8601] compatible date/time strings.
* `producer_timestamp` (timestamp): is the time of catalog submission from the master to PuppetDB. This field is currently populated by the master. Timestamps are always [ISO-8601][8601] compatible date/time strings.
* `receive_time` (timestamp): is the time at which PuppetDB received the report. Timestamps are always [ISO-8601][8601] compatible date/time strings.
......
......@@ -20,6 +20,7 @@ otherwise noted, `null` is not allowed anywhere in the report.
"configuration_version": <string>,
"start_time": <datetime>,
"end_time": <datetime>,
"producer_timestamp": <datetime>,
"resource_events": [<resource_event>, <resource_event>, ...],
"metrics": [<metric>, <metric>, ...],
"logs": [<log>, <log>, ...],
......@@ -45,11 +46,15 @@ to generate the original report data. This is a constant defined by puppet.
specific catalog for a node to a specific puppet run. This value is
generated by puppet.
`"start_time"` is the time at which the puppet run began; see more details about
the `datetime` format below.
`"start_time"` is the time on the agent at which the puppet run began; see more
details about the `datetime` format below.
`"end_time"` is the time at which the puppet run completed; see more details about
the `datetime` format below.
`"end_time"` is the time on the agent at which the puppet run completed; see
more details about the `datetime` format below.
`"producer_timestamp"` is the time of catalog submission from the master to
PuppetDB. This field is currently populated by the master. See more details
about the `datetime` format below.
`"transaction_uuid"` is a string used to identify a puppet run. It can be used to
match a report with the catalog that was used for the run. This field may be `null`.
......
......@@ -45,6 +45,7 @@ Puppet::Reports.register_report(:puppetdb) do
"puppet_version" => puppet_version,
"report_format" => report_format,
"configuration_version" => configuration_version.to_s,
"producer_timestamp" => Puppet::Util::Puppetdb.to_wire_time(Time.now),
"start_time" => Puppet::Util::Puppetdb.to_wire_time(time),
"end_time" => Puppet::Util::Puppetdb.to_wire_time(time + run_duration),
"resource_events" => resource_events,
......
......@@ -24,20 +24,29 @@ describe processor do
let(:http) { mock "http" }
let(:httpok) { Net::HTTPOK.new('1.1', 200, '') }
def without_producer_timestamp(json_body)
parsed = JSON.parse(json_body)
parsed["payload"].delete("producer_timestamp")
parsed.to_json
end
it "should POST the report command as a URL-encoded JSON string" do
httpok.stubs(:body).returns '{"uuid": "a UUID"}'
subject.stubs(:run_duration).returns(10)
payload = {
expected_body = {
:command => Puppet::Util::Puppetdb::CommandNames::CommandStoreReport,
:version => 5,
:payload => subject.send(:report_to_hash),
:payload => subject.send(:report_to_hash)
}.to_json
Puppet::Network::HttpPool.expects(:http_instance).returns(http)
http.expects(:post).with {|path, body, headers|
expect(path).to include(Puppet::Util::Puppetdb::Command::CommandsUrl)
expect(body).to eq(payload)
# producer_timestamp is generated at submission time, so remove it from
# the comparison
expect(without_producer_timestamp(body)).to eq(without_producer_timestamp(expected_body))
}.returns(httpok)
subject.process
......
......@@ -139,7 +139,8 @@
(assoc report
"configuration_version" (ks/uuid)
"start_time" (time/now)
"end_time" (time/now)))
"end_time" (time/now)
"producer_timestamp" (time/now)))
(defn randomize-map-leaf
"Randomizes a fact leaf based on a percentage provided with `rp`."
......@@ -172,7 +173,7 @@
(defn update-factset
"Updates the producer_timestamp to be current, and randomly updates the leaves
of the factset based on a percentage provided in `rand-percentage`."
[factset rand-percentage ]
[factset rand-percentage]
(-> factset
(assoc "producer_timestamp" (time/now))
(update-in ["values"] (partial randomize-map-leaves rand-percentage))))
......@@ -195,7 +196,7 @@
(let [base-url {:protocol "http" :host puppetdb-host :port puppetdb-port}
catalog (some-> catalog update-catalog (maybe-tweak-catalog rand-percentage))
report (some-> report update-report-run-fields)
factset (some-> factset (update-factset rand-percentage ))]
factset (some-> factset (update-factset rand-percentage))]
;; Submit the catalog and reports in separate threads, so as to not
;; disturb the world-loop and otherwise distort the space-time continuum.
(when catalog
......@@ -233,7 +234,7 @@
(let [base-url {:protocol "http" :host puppetdb-host :port puppetdb-port}
catalog (some-> catalog (maybe-tweak-catalog rand-percentage))
report (some-> report update-report-run-fields)
factset (some-> factset (update-factset factset rand-percentage))]
factset (some-> factset (update-factset rand-percentage))]
(when catalog
(client/submit-catalog base-url 6 (json/generate-string catalog)))
(when report
......
......@@ -235,15 +235,13 @@
[version db {:keys [payload annotations]}]
(let [id (:id annotations)
received-timestamp (:received annotations)
{:keys [certname puppet_version] :as report}
(->> payload
(s/validate report/report-wireformat-schema)
upon-error-throw-fatality)]
upon-error-throw-fatality)
producer-timestamp (to-timestamp (:producer_timestamp payload (now)))]
(jdbc/with-transacted-connection db
;; This is using received-timestamp because the store-report command
;; doesn't yet have a producer_timestamp field. It will be added soon.
(scf-storage/maybe-activate-node! certname received-timestamp)
(scf-storage/maybe-activate-node! certname producer-timestamp)
(scf-storage/add-report! report received-timestamp))
(log/infof "[%s] [%s] puppet v%s - %s"
id (command-names :store-report)
......
......@@ -250,6 +250,7 @@
"configuration_version" "reports"
"start_time" "reports"
"end_time" "reports"
"producer_timestamp" "reports"
"receive_time" "reports"
"transaction_uuid" "reports"
"environment" "reports"
......
......@@ -50,6 +50,7 @@
:report_format
:start_time
:end_time
:producer_timestamp
:noop
:transaction_uuid
:status
......
......@@ -331,6 +331,9 @@
"end_time" {:type :timestamp
:queryable? true
:field :reports.end_time}
"producer_timestamp" {:type :timestamp
:queryable? true
:field :reports.producer_timestamp}
"metrics" {:type :json
:queryable? false
:field {:select [(h/row-to-json :t)]
......
......@@ -43,6 +43,7 @@
:configuration_version s/Str
:start_time pls/Timestamp
:end_time pls/Timestamp
:producer_timestamp pls/Timestamp
:resource_events [resource-event-wireformat-schema]
:noop (s/maybe s/Bool)
:transaction_uuid (s/maybe s/Str)
......@@ -106,6 +107,7 @@
(s/optional-key :receive_time) pls/Timestamp
(s/optional-key :start_time) pls/Timestamp
(s/optional-key :end_time) pls/Timestamp
(s/optional-key :producer_timestamp) pls/Timestamp
(s/optional-key :noop) (s/maybe s/Bool)
(s/optional-key :report_format) s/Int
(s/optional-key :configuration_version) s/Str
......
......@@ -151,7 +151,7 @@
configuration version, timestamps, events).
"
[{:keys [certname puppet_version report_format configuration_version
start_time end_time resource_events transaction_uuid] :as report}]
start_time end_time producer_timestamp resource_events transaction_uuid] :as report}]
(generic-identity-hash
{:certname certname
:puppet_version puppet_version
......@@ -159,5 +159,6 @@
:configuration_version configuration_version
:start_time start_time
:end_time end_time
:producer_timestamp producer_timestamp
:resource_events (sort (map resource-event-identity-string resource_events))
:transaction_uuid transaction_uuid}))
......@@ -1331,6 +1331,13 @@
"ALTER TABLE fact_values RENAME COLUMN value_json TO value"
"ALTER TABLE fact_values ALTER COLUMN value_json RENAME TO value"))))
(defn add-producer-timestamp-to-reports []
(sql/do-commands
"ALTER TABLE reports ADD producer_timestamp TIMESTAMP WITH TIME ZONE"
"UPDATE reports SET producer_timestamp=end_time"
"ALTER TABLE reports ALTER COLUMN producer_timestamp SET NOT NULL"
"CREATE INDEX idx_reports_producer_timestamp ON reports(producer_timestamp)"))
(def migrations
"The available migrations, as a map from migration version to migration function."
{1 initialize-store
......@@ -1363,7 +1370,8 @@
28 lift-fact-paths-into-facts
29 version-2yz-to-300-migration
30 add-expired-to-certnames
31 coalesce-fact-values})
31 coalesce-fact-values
32 add-producer-timestamp-to-reports})
(def desired-schema-version (apply max (keys migrations)))
......
......@@ -1101,6 +1101,7 @@
(-> report
(update-in [:start_time] to-timestamp)
(update-in [:end_time] to-timestamp)
(update-in [:producer_timestamp] to-timestamp)
(update-in [:resource_events] #(map normalize-resource-event %))))
(defn convert-containment-path
......@@ -1122,7 +1123,7 @@
update-latest-report? :- s/Bool]
(time! (:store-report performance-metrics)
(let [{:keys [puppet_version certname report_format configuration_version
start_time end_time resource_events transaction_uuid environment
producer_timestamp start_time end_time resource_events transaction_uuid environment
status noop metrics logs] :as report} (normalize-report orig-report)
report-hash (shash/report-identity-hash report)]
(sql/transaction
......@@ -1137,6 +1138,7 @@
:certname certname
:report_format report_format
:configuration_version configuration_version
:producer_timestamp producer_timestamp
:start_time start_time
:end_time end_time
:receive_time (to-timestamp received-timestamp)
......@@ -1149,15 +1151,17 @@
(update-latest-report! certname)))))))
(defn delete-reports-older-than!
"Delete all reports in the database which have an `end-time` that is prior to
"Delete all reports in the database which have an `producer-timestamp` that is prior to
the specified date/time."
[time]
{:pre [(kitchensink/datetime? time)]}
(when (not (sutils/postgres?))
(sql/update-values :certnames ["latest_report_id in (select id from reports where end_time < ?)"
;; there's an ON DELETE SET NULL foreign key constraint in postgres for
;; this, but we can't do that in hsqldb
(sql/update-values :certnames ["latest_report_id in (select id from reports where producer_timestamp < ?)"
(to-timestamp time)]
{:latest_report_id nil}))
(sql/delete-rows :reports ["end_time < ?" (to-timestamp time)]))
(sql/delete-rows :reports ["producer_timestamp < ?" (to-timestamp time)]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Database support/deprecation
......
......@@ -9,6 +9,7 @@
:configuration_version "a81jasj123"
:start_time "2011-01-01T12:00:00-03:00"
:end_time "2011-01-01T12:10:00-03:00"
:producer_timestamp "2011-01-01T12:11:00-03:00"
:environment "DEV"
:status "unchanged"
:noop false
......@@ -96,6 +97,7 @@
:configuration_version "bja3985a23"
:start_time "2013-08-28T19:00:00-03:00"
:end_time "2013-08-28T19:10:00-03:00"
:producer_timestamp "2013-08-28T19:11:00-03:00"
:environment "DEV"
:status "unchanged"
:noop true
......@@ -180,6 +182,7 @@
:configuration_version "a81jasj123"
:start_time "2011-01-03T12:00:00-03:00"
:end_time "2011-01-03T12:10:00-03:00"
:producer_timestamp "2011-01-03T12:11:00-03:00"
:environment "DEV"
:status "unchanged"
:noop false
......@@ -264,6 +267,7 @@
:configuration_version "a81jasj123"
:start_time "2011-01-03T12:00:00-03:00"
:end_time "2011-01-03T12:10:00-03:00"
:producer_timestamp "2011-01-03T12:11:00-03:00"
:environment "DEV"
:status "unchanged"
:noop false
......
......@@ -58,9 +58,9 @@
(let [request (get-request endpoint (json/generate-string query))
{:keys [status body]} (*app* request)
actual-result (parse-result body)]
(is (= (count actual-result) (count expected-results)))
(is (= (set actual-result) expected-results))
(is (= status http/status-ok))))
(is (= (count expected-results) (count actual-result)))
(is (= expected-results (set actual-result)))
(is (= http/status-ok status))))
(defn munge-event-values
"Munge the event values that we get back from the web to a format suitable
......@@ -340,7 +340,7 @@
:new_value nil
:containing_class "Foo"
:report_receive_time "2014-04-16T12:44:40.978Z"
:report "a32722b44f0852d9a16d326414c16a6941b9678f"
:report "99ec099bed6dfb9bff2c7df7828270e95f590147"
:resource_title "hi"
:property nil
:file "bar"
......@@ -365,7 +365,7 @@
:new_value nil
:containing_class "Foo"
:report_receive_time "2014-04-16T12:44:40.978Z"
:report "a32722b44f0852d9a16d326414c16a6941b9678f"
:report "99ec099bed6dfb9bff2c7df7828270e95f590147"
:resource_title "hi"
:property nil
:file "bar"
......@@ -392,7 +392,7 @@
:new_value nil
:containing_class "Foo"
:report_receive_time "2014-04-16T12:44:40.978Z"
:report "a32722b44f0852d9a16d326414c16a6941b9678f"
:report "99ec099bed6dfb9bff2c7df7828270e95f590147"
:resource_title "hi"
:property nil
:file "bar"
......
......@@ -51,7 +51,7 @@
;; them to be coerced to dates and then back to strings, which normalizes
;; the timezone so that it will match the value returned form the db.
to-string
[:start_time :end_time]
[:start_time :end_time :producer_timestamp]
;; the response won't include individual events, so we need to pluck those
;; out of the example report object before comparison
(-> report
......
......@@ -30,7 +30,7 @@
[record]
(kitchensink/mapvals
to-string
[:start_time :end_time]
[:start_time :end_time :producer_timestamp]
record))
(defn munge-expected-report
......
......@@ -140,6 +140,7 @@
:configuration_version "asdffdsa"
:start_time "2012-03-01-12:31:11.123"
:end_time "2012-03-01-12:31:31.123"
:producer_timestamp "2012-03-01-1:31:51.123"
:resource_events [
{:type "Type"
:title "title"
......@@ -148,8 +149,8 @@
:line 15}]}]
(testing "should return sorted predictable string output"
(is (= (report-identity-hash sample)
"7fddeb9eb1f4469acb9ea6c5d1bea15f8654326b")))
(is (= "3016159f704726b486f8b42309773ec625e2f3b7"
(report-identity-hash sample))))
(testing "should return the same value twice"
(is (= (report-identity-hash sample)
......
......@@ -707,7 +707,8 @@
(let [timestamp (now)
report (-> (:basic reports)
(assoc :environment "ENV2")
(assoc :end_time (to-string (-> 5 days ago))))
(assoc :end_time (to-string (-> 5 days ago)))
(assoc :producer_timestamp (to-string (-> 4 days ago))))
report-hash (shash/report-identity-hash report)
certname (:certname report)]
(store-example-report! report timestamp)
......@@ -1261,7 +1262,7 @@
(deftest report-sweep-nullifies-latest-report
(testing "ensure that if the latest report is swept, latest_report_id is updated to nil"
(let [report1 (assoc (:basic reports) :end_time (-> 12 days ago))
report2 (assoc (:basic reports) :certname "bar.local" :end_time (now))]
report2 (assoc (:basic reports) :certname "bar.local" :end_time (now) :producer_timestamp (now))]
(add-certname! "foo.local")
(add-certname! "bar.local")
(store-example-report! report1 (-> 12 days ago))
......@@ -1354,9 +1355,13 @@
(deftest report-cleanup
(testing "should delete reports older than the specified age"
(let [report1 (assoc report :end_time (to-string (-> 5 days ago)))
(let [report1 (assoc report
:end_time (to-string (-> 5 days ago))
:producer_timestamp (to-string (-> 5 days ago)))
report1-hash (:hash (store-example-report! report1 timestamp))
report2 (assoc report :end_time (to-string (-> 2 days ago)))
report2 (assoc report
:end_time (to-string (-> 2 days ago))
:producer_timestamp (to-string (-> 2 days ago)))
report2-hash (:hash (store-example-report! report2 timestamp))
certname (:certname report1)
_ (delete-reports-older-than! (-> 3 days ago))
......
......@@ -46,6 +46,7 @@
(clojure.walk/stringify-keys)
(update-in ["start_time"] time-coerce/to-string)
(update-in ["end_time"] time-coerce/to-string)
(update-in ["producer_timestamp"] time-coerce/to-string)
(update-in ["resource_events"] munge-events-for-comparison)
(dissoc "hash")
(dissoc "receive_time")))
......@@ -89,7 +90,7 @@
(kitchensink/mapvals
;; we need to map the datetime fields to timestamp objects for comparison
time-coerce/to-timestamp
[:start_time :end_time]
[:start_time :end_time :producer_timestamp]
;; the response won't include individual events, so we need to pluck those
;; out of the example report object before comparison
example-report))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment