Commit 09735b80 authored by Wyatt Alt's avatar Wyatt Alt

(PDB-2753) add noop pending to reports and nodes

Thread the new "noop_pending" Puppet report property through PDB and expose
via /reports and /nodes (for the latest report).
parent 54871e11
......@@ -126,6 +126,7 @@ payload of this command.
### "store report", version 8
* The nullable `producer` property has been added.
* The nullable `noop_pending` property has been added.
The payload is expected to be a report, containing events that occurred on
Puppet resources. It is structured as a JSON object, conforming to the
......@@ -64,6 +64,9 @@ The below fields are allowed as filter criteria and are returned in all response
* `latest_report_noop` (boolean): indicates whether the most recent report for
the node was a noop run.
* `latest_report_noop_pending` (boolean): indicates whether the most recent
report for the node contained noop events.
* `cached_catalog_status` (string): Cached catalog status of the
last puppet run for the node. Possible values are `explicitly_requested`,
`on_failure`, `not_used` or `null`.
......@@ -105,6 +108,7 @@ The response is a JSON array of hashes, where each hash has the form:
"report_environment": <string or null>,
"latest_report_status": <string>,
"latest_report_noop": <boolean>,
"latest_report_noop_pending": <boolean>,
"latest_report_hash": <string>
......@@ -162,6 +166,7 @@ of `["=", "certname", "<NODE>"]`.
"catalog_timestamp" : "2015-06-19T23:03:43.007Z",
"latest_report_status": "success",
"latest_report_noop": false,
"latest_report_noop_pending": true,
"latest_report_hash": "2625d1b601e98ed1e281ccd79ca8d16b9f74fea6"
......@@ -70,6 +70,10 @@ responses.
* `noop` (Boolean): a flag indicating whether the report was produced by a noop
* `noop_pending` (Boolean): a flag indicating whether the report contains noop
events (these can arise from use of `--noop` or from resources with the
`noop` parameter set to true).
* `puppet_version` (string): the version of Puppet that generated the report.
* `report_format` (number): the version number of the report format that Puppet
......@@ -139,6 +143,7 @@ is of the form:
"transaction_uuid": <string to identify puppet run>,
"status": <status of node after report's associated puppet run>,
"noop": <boolean flag indicating noop run>,
"noop_pending": <boolean flag indicating presence of noop events>
"environment": <report environment>,
"configuration_version": <catalog identifier>,
"certname": <node name>,
......@@ -256,6 +261,7 @@ Query for all reports:
"transaction_uuid" : "9a7070e9-840f-446d-b756-6f19bf2e2efc",
"puppet_version" : "3.7.4",
"noop" : false,
"noop_pending": true,
"report_format" : 4,
"start_time" : "2015-02-19T16:23:09.810Z",
"end_time" : "2015-02-19T16:23:10.287Z",
......@@ -27,7 +27,8 @@ noted, `null` is not allowed anywhere in the report.
"code_id": <string>,
"cached_catalog_status": <string>,
"status": <string>,
"noop": <boolean>
"noop": <boolean>,
"noop_pending": <boolean>
All keys are mandatory unless otherwise noted, though values that are lists may be empty lists.
......@@ -76,6 +77,10 @@ error or not. This field may be `null`.
`"noop"` is a flag that indicates whether the report was produced with a `--noop` run.
`"noop_pending"` is a flag that indicates whether the report contained "noop"
events. These may result from use of the `--noop` flag, or from resources
tagged with the `noop` parameter. This field may be `null`.
`"resources"` is an array of objects of the following form:
......@@ -40,10 +40,11 @@ Puppet::Reports.register_report(:puppetdb) do
resources = build_resources_list
is_noop = defined?(noop) ? noop : resources.any? { |rs| has_noop_event?(rs) } && resources.none? { |rs| has_enforcement_event?(rs) }
defaulted_catalog_uuid = defined?(catalog_uuid) ? catalog_uuid : transaction_uuid
defaulted_code_id = defined?(code_id) ? code_id : nil
defaulted_cached_catalog_status = defined?(cached_catalog_status) ? cached_catalog_status : nil
defaulted_noop_pending = defined?(noop_pending) ? noop_pending : nil
"certname" => host,
"puppet_version" => puppet_version,
......@@ -56,6 +57,7 @@ Puppet::Reports.register_report(:puppetdb) do
"transaction_uuid" => transaction_uuid,
"status" => status,
"noop" => is_noop,
"noop_pending" => defaulted_noop_pending,
"logs" => build_logs_list,
"metrics" => build_metrics_list,
"resources" => resources,
......@@ -74,6 +74,7 @@ describe processor do
if defined?(subject.catalog_uuid) then
subject.catalog_uuid = 'bde432'
result = subject.send(:report_to_hash)
result["transaction_uuid"].should == 'abc123'
......@@ -103,6 +104,18 @@ describe processor do
result["producer"].should == "foo"
it "should include noop_pending or nil" do
if defined?(subject.noop_pending) then
subject["noop_pending"] = false
result = subject.send(:report_to_hash)
if defined?(subject.noop_pending) then
result["noop_pending"].should == false
result["noop_pending"].should == nil
it "should include the cached_catalog_status or nil" do
if defined?(subject.cached_catalog_status) then
subject.cached_catalog_status = 'not_used'
......@@ -143,6 +143,9 @@
"latest_report_noop" {:type :boolean
:queryable? true
:field :reports.noop}
"latest_report_noop_pending" {:type :boolean
:queryable? true
:field :reports.noop_pending}
"latest_report_status" {:type :string
:queryable? true
:field :report_statuses.status}
......@@ -444,6 +447,9 @@
"certname" {:type :string
:queryable? true
:field :reports.certname}
"noop_pending" {:type :boolean
:queryable? true
:field :reports.noop_pending}
"puppet_version" {:type :string
:queryable? true
:field :reports.puppet_version}
......@@ -4,6 +4,7 @@
Functions that handle conversion of reports from wire format to
internal PuppetDB format, including validation."
(:require [schema.core :as s]
[clojure.set :as set]
[puppetlabs.puppetdb.schema :as pls]
[puppetlabs.puppetdb.utils :as utils]
[com.rpl.specter :as sp]))
......@@ -53,6 +54,7 @@
:producer (s/maybe s/Str)
:resources [resource-wireformat-schema]
:noop (s/maybe s/Bool)
:noop_pending (s/maybe s/Bool)
:transaction_uuid (s/maybe s/Str)
:catalog_uuid (s/maybe s/Str)
:code_id (s/maybe s/Str)
......@@ -64,7 +66,7 @@
(def report-v7-wireformat-schema
(-> report-wireformat-schema
(dissoc :producer)))
(dissoc :producer :noop_pending)))
(def report-v6-wireformat-schema
(-> report-v7-wireformat-schema
......@@ -140,6 +142,7 @@
(s/optional-key :producer_timestamp) pls/Timestamp
(s/optional-key :producer) (s/maybe s/Str)
(s/optional-key :noop) (s/maybe s/Bool)
(s/optional-key :noop_pending) (s/maybe s/Bool)
(s/optional-key :report_format) s/Int
(s/optional-key :configuration_version) s/Str
(s/optional-key :resources) (s/maybe resources-expanded-query-schema)
......@@ -163,14 +166,22 @@
:report :certname :containing_class :configuration_version
:run_start_time :run_end_time :report_receive_time :environment))))
(defn report-query->wire-v5
(defn generic-query->wire-transform
"Dissociate query-only and pe-only fields and replace href fields with the
corresponding data key."
(-> report
(dissoc :hash :receive_time :resources)
(update :resource_events resource-events-query->wire-v5)
(update :metrics :data)
(update :logs :data)))
(defn report-query->wire-v5
(-> report
(dissoc :noop_pending)
(update :resource_events resource-events-query->wire-v5)))
(pls/defn-validated reports-query->wire-v5 :- [report-v5-wireformat-schema]
[reports :- [report-query-schema]]
(map report-query->wire-v5 reports))
......@@ -194,10 +205,11 @@
:let [events (mapv #(dissoc % :file :line :resource_type :resource_title :containment_path) resource-events)]]
(assoc resource :events events))))
(defn wire-v7->wire-v8
[{:keys [transaction_uuid] :as report}]
(utils/assoc-when report
:producer nil))
(defn wire-v7->wire-v8
(utils/assoc-when report
:noop_pending nil
:producer nil))
(defn wire-v6->wire-v8
[{:keys [transaction_uuid] :as report}]
......@@ -211,7 +223,7 @@
(-> report
(update :resource_events resource-events-v5->resources)
(clojure.set/rename-keys {:resource_events :resources})
(set/rename-keys {:resource_events :resources})
(defn wire-v4->wire-v8
......@@ -234,8 +246,10 @@
(pls/defn-validated report-query->wire-v8 :- report-wireformat-schema
[report :- report-query-schema]
(-> report
(update :resource_events (comp resource-events-v5->resources
(set/rename-keys {:resource_events :resources})))
(defn reports-query->wire-v8 [reports]
(map report-query->wire-v8 reports))
......@@ -1056,6 +1056,12 @@
"DROP INDEX idx_certnames_latest_report_id"))
(defn add-noop-pending-to-reports
"ALTER TABLE reports ADD COLUMN noop_pending boolean"
"CREATE INDEX idx_reports_noop_pending on reports using btree (noop_pending) where (noop_pending = true)"))
(def migrations
"The available migrations, as a map from migration version to migration function."
{28 init-through-2-3-8
......@@ -1080,7 +1086,8 @@
44 add-catalog-uuid-to-reports-and-catalogs
45 index-certnames-latest-report-id
46 drop-certnames-latest-id-index
47 add-producer-to-reports-catalogs-and-factsets})
47 add-producer-to-reports-catalogs-and-factsets
48 add-noop-pending-to-reports})
(def desired-schema-version (apply max (keys migrations)))
......@@ -1247,7 +1247,7 @@
(let [{:keys [puppet_version certname report_format configuration_version producer
producer_timestamp start_time end_time transaction_uuid environment
status noop metrics logs resources resource_events catalog_uuid
code_id cached_catalog_status]
code_id cached_catalog_status noop_pending]
:as report} (normalize-report orig-report)
report-hash (shash/report-identity-hash report)]
(jdbc/with-db-transaction []
......@@ -1261,6 +1261,7 @@
:logs (sutils/munge-jsonb-for-storage logs)
:resources (sutils/munge-jsonb-for-storage resources)
:noop noop
:noop_pending noop_pending
:puppet_version puppet_version
:certname certname
:report_format report_format
......@@ -1315,7 +1315,7 @@
(def v7-report
(-> v8-report
(dissoc :producer)))
(dissoc :producer :noop_pending)))
(def v6-report
(-> v7-report
......@@ -17,6 +17,7 @@
:environment "DEV"
:status "unchanged"
:noop false
:noop_pending true
{:href ""
......@@ -108,6 +109,7 @@
:environment "DEV"
:status "unchanged"
:noop true
:noop_pending true
{:href ""
......@@ -196,6 +198,7 @@
:environment "DEV"
:status "unchanged"
:noop false
:noop_pending false
{:href ""
......@@ -284,6 +287,7 @@
:environment "DEV"
:status "unchanged"
:noop false
:noop_pending true
{:href ""
......@@ -44,6 +44,7 @@
(is (= #{:certname :deactivated :expired :catalog_timestamp :facts_timestamp :report_timestamp
:catalog_environment :facts_environment :report_environment
:latest_report_status :latest_report_hash :latest_report_noop
:cached_catalog_status} (keyset res))
(str "Query was: " query))
(is (= (set expected) (set (mapv :certname result)))
......@@ -40,7 +40,7 @@
(doseq [field ["certname" "hash" "puppet_version" "report_format"
"configuration_version" "start_time" "end_time"
"transaction_uuid" "status" "producer"]
"transaction_uuid" "status" "producer" "noop_pending"]
:let [field-kwd (keyword field)]]
(testing (format "should return all reports for a %s" field)
(let [result (query-response method endpoint ["=" field (get basic-with-hash field-kwd)])
......@@ -39,7 +39,7 @@
(def v7-example-report
(-> v8-example-report
(dissoc :producer)))
(dissoc :producer :noop_pending)))
(def v6-example-report
(-> v7-example-report
......@@ -90,6 +90,6 @@
v3-report (dissoc v4-example-report :status)
v8-report (wire-v3->wire-v8 v3-report current-time)]
(is (s/validate report-v3-wireformat-schema v3-report))
(is (s/validate report-wireformat-schema (wire-v3->wire-v8 v3-report current-time)))
(is (s/validate report-wireformat-schema v8-report))
(is (= current-time (:producer_timestamp v8-report)))
(is (nil? (:status v8-report)))))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment