Commit 5abc90eb authored by wkalt's avatar wkalt

(PDB-1192) add metrics and logs support

This adds two fields, logs and metrics, to the reports response to contain
logs and metrics passed from Puppet in the terminus. This is part of the ongoing
work to retire consoleDB.
parent 32ff1dee
......@@ -45,15 +45,41 @@ test_name "basic validation of puppet report submission" do
assert_equal("Hi #{agent.node_name}", event["new_value"], "new_value doesn't match!")
end
step "ensure that noop is false when run without --noop" do
step "do a run without noop" do
run_agents_with_new_site_pp(master, manifest)
sleep_until_queue_empty database
end
agents.each do |agent|
result = on database, %Q|curl -G http://localhost:8080/v4/reports --data 'query=["=",%20"certname",%20"#{agent.node_name}"]' --data 'order_by=[{"field":"receive_time","order":"desc"}]'|
reports = JSON.parse(result.stdout)
report = reports[0]
assert_equal(false, report["noop"], "noop does not match!")
metrics = report["metrics"]
logs = report["logs"]
step "ensure that noop is false for #{agent}" do
assert_equal(false, report["noop"], "noop does not match!")
end
step "ensure that metrics check out for #{agent}" do
total_events = metrics.detect {|m| m["name"] == "total" && m["category"] == "events"}
total_changes = metrics.detect {|m| m["name"] == "total" && m["category"] == "changes"}
resources_changed = metrics.detect {|m| m["name"] == "changed" && m["category"] == "resources"}
assert(total_events["value"] == 1, "metrics do not match")
assert(total_changes["value"] == 1, "metrics do not match")
assert(total_changes["value"] == 1, "metrics do not match")
end
step "ensure that logs check out for #{agent}" do
line_change = logs.detect {|l| l["line"] == 3}
notice_level = logs.select {|l| l["level"] == "notice"}
info_level = logs.select {|l| l["level"] == "info"}
assert(line_change["tags"].to_set == ["notice", "notify", "hi", "class"].to_set,
"tags of logs do not match!")
assert(notice_level.count == 3, "notice level count does not match!")
assert(info_level.count == 5, "info level count does not match!")
end
end
end
......@@ -86,7 +86,9 @@ is of the form:
"environment": <report environment>,
"configuration_version": <catalog identifier>,
"certname": <node name>,
"resource_events": [<resource event>]
"resource_events": [<resource event>],
"metrics" : [<metric>],
"logs" : [<log>]
}
Resource event objects are of the following form:
......@@ -101,16 +103,46 @@ Resource event objects are of the following form:
"old_value": <old value of resource property>,
"message": <description of what happened during event>,
"file": <manifest file containing resource definition>,
"line": <line in manifest file on which resource is defined>
"line": <line in manifest file on which resource is defined>,
"containment_path": <containment heirarchy of resource within catalog>
}
The metrics field may be either null, in the case of reports submitted without metrics,
or a JSON array of objects like so:
{
"category" : <category of metric ("resources", "time", "changes", or "events")>,
"name" : <name of the metric>,
"value" : <value of the metric (double precision)>
}
The logs field may be either null, in the case of reports submitted without
logs, or a JSON array of objects like so, each corresponding to a log line in a
Puppet run:
{
"file" : <file of resource declaration>,
"line" : <line of resource declaration>,
"level" : <log level>,
"message" : <log message>,
"source" : <log source>,
"tags" : [<resource tag>],
"time" : <log line timestamp>
}
File and line may each be null if the log does not concern a resource.
**Note on fields that allow `NULL` values**
In the resource_event schema above, `containment_path`, `new_value`, `old_value`, `property`, `file`, `line`, `status`, and `message` may all be null.
**Note on querying resource events**
The `reports` endpoint does not support querying on the value of `resource_events`, but the same information can be be accessed by querying the `events` endpoint for events with field `report` equal to a given report's `hash`.
**Note on querying resource events, metrics, and logs**
The `reports` endpoint does not support querying on the value of `resource_events`, `logs`,
or `metrics`. In the case of `resource_events` the same information can be accessed by querying the `events` endpoint for events with field `report` equal to a given report's `hash`.
Making metrics and logs queryable may be the target of future work.
### Examples
......@@ -122,42 +154,152 @@ Query for all reports:
curl -G 'http://localhost:8080/v4/reports'
[ {
"hash" : "89944d0dcac56d3ee641ca9b69c54b1c15ef01fe",
"puppet_version" : "3.7.3",
"receive_time" : "2014-12-24T00:00:50.716Z",
"report_format" : 4,
"start_time" : "2014-12-24T00:00:49.211Z",
"end_time" : "2014-12-24T00:00:49.705Z",
"transaction_uuid" : "af4fb9ad-b267-4e0b-a295-53eba6b139b7",
"status" : "changed",
"receive_time" : "2015-02-19T16:23:11.034Z",
"hash" : "32c821673e647b0650717db467abc51d9949fd9a",
"transaction_uuid" : "9a7070e9-840f-446d-b756-6f19bf2e2efc",
"puppet_version" : "3.7.4",
"noop" : false,
"environment" : "production",
"configuration_version" : "1419379250",
"certname" : "foo.com",
"report_format" : 4,
"start_time" : "2015-02-19T16:23:09.810Z",
"end_time" : "2015-02-19T16:23:10.287Z",
"resource_events" : [ {
"containment_path" : [ "Stage[main]", "Main", "Notify[hi]" ],
"new_value" : "\"Hi world\"",
"resource_title" : "hi",
"new_value" : "hi world",
"property" : "message",
"file" : "/home/wyatt/.puppet/manifests/site.pp",
"old_value" : "\"absent\"",
"line" : 3,
"status" : "changed",
"old_value" : "absent",
"line" : 7,
"resource_type" : "Notify",
"timestamp" : "2014-12-24T00:00:50.522Z",
"message" : "defined 'message' as 'Hi world'"
"status" : "success",
"resource_title" : "hiloo",
"timestamp" : "2015-02-19T16:23:10.768Z",
"containment_path" : [ "Stage[main]", "Main", "Notify[hiloo]" ],
"message" : "defined 'message' as 'hi world'"
}, {
"containment_path" : [ "Stage[main]", "Main", "File[/home/wyatt/Desktop/foo]" ],
"new_value" : "\"file\"",
"resource_title" : "/home/wyatt/Desktop/foo",
"property" : "ensure",
"new_value" : "hi world",
"property" : "message",
"file" : "/home/wyatt/.puppet/manifests/site.pp",
"old_value" : "\"absent\"",
"line" : 7,
"status" : "changed",
"resource_type" : "File",
"timestamp" : "2014-12-24T00:00:50.514Z",
"message" : "defined content as '{md5}207995b58ba1956b97028ebb2f8caeba'"
"old_value" : "absent",
"line" : 3,
"resource_type" : "Notify",
"status" : "success",
"resource_title" : "hi",
"timestamp" : "2015-02-19T16:23:10.767Z",
"containment_path" : [ "Stage[main]", "Main", "Notify[hi]" ],
"message" : "defined 'message' as 'hi world'"
} ],
"status" : "changed",
"configuration_version" : "1424362990",
"environment" : "production",
"certname" : "desktop.localdomain",
"metrics" : [ {
"category" : "resources",
"name" : "changed",
"value" : 2
}, {
"category" : "resources",
"name" : "failed",
"value" : 0
}, {
"category" : "resources",
"name" : "failed_to_restart",
"value" : 0
}, {
"category" : "resources",
"name" : "out_of_sync",
"value" : 2
}, {
"category" : "resources",
"name" : "restarted",
"value" : 0
}, {
"category" : "resources",
"name" : "scheduled",
"value" : 0
}, {
"category" : "resources",
"name" : "skipped",
"value" : 0
}, {
"category" : "resources",
"name" : "total",
"value" : 9
}, {
"category" : "time",
"name" : "config_retrieval",
"value" : 0.476064209
}, {
"category" : "time",
"name" : "filebucket",
"value" : 3.8841E-5
}, {
"category" : "time",
"name" : "notify",
"value" : 7.54224E-4
}, {
"category" : "time",
"name" : "schedule",
"value" : 2.0780000000000004E-4
}, {
"category" : "time",
"name" : "total",
"value" : 0.47706507400000003
}, {
"category" : "changes",
"name" : "total",
"value" : 2
}, {
"category" : "events",
"name" : "failure",
"value" : 0
}, {
"category" : "events",
"name" : "success",
"value" : 2
}, {
"category" : "events",
"name" : "total",
"value" : 2
} ],
"logs" : [ {
"file" : null,
"line" : null,
"level" : "info",
"message" : "Caching catalog for mbp.local",
"source" : "//mbp.local/Puppet",
"tags" : [ "info" ],
"time" : "2015-02-26T16:27:48.416642000-08:00"
}, {
"file" : null,
"line" : null,
"level" : "info",
"message" : "Applying configuration version '1424996868'",
"source" : "//mbp.local/Puppet",
"tags" : [ "info" ],
"time" : "2015-02-26T16:27:48.474162000-08:00"
}, {
"file" : null,
"line" : null,
"level" : "notice",
"message" : "Hi mbp.local",
"source" : "//mbp.local/Puppet",
"tags" : [ "notice" ],
"time" : "2015-02-26T16:27:48.475656000-08:00"
}, {
"file" : "/Users/wyatt/.puppet/manifests/site.pp",
"line" : 3,
"level" : "notice",
"message" : "defined 'message' as 'Hi mbp.local'",
"source" : "//mbp.local//Stage[main]/Main/Notify[hi]/message",
"tags" : [ "notice", "notify", "hi", "class" ],
"time" : "2015-02-26T16:27:48.475825000-08:00"
}, {
"file" : null,
"line" : null,
"level" : "notice",
"message" : "Finished catalog run in 0.01 seconds",
"source" : "//mbp.local/Puppet",
"tags" : [ "notice" ],
"time" : "2015-02-26T16:27:48.483317000-08:00"
} ]
} ]
......
......@@ -5,6 +5,7 @@ canonical: "/puppetdb/latest/api/wire_format/report_format_v5.html"
---
[puppetreportformat]: http://docs.puppetlabs.com/puppet/latest/reference/format_report.html
[reportsv4]: ../query/v4/reports.markdown
## Report interchange format
......@@ -20,16 +21,20 @@ otherwise noted, `null` is not allowed anywhere in the report.
"start_time": <datetime>,
"end_time": <datetime>,
"resource_events": [<resource_event>, <resource_event>, ...],
"metrics": [<metric>, <metric>, ...],
"logs": [<log>, <log>, ...],
"transaction_uuid": <string>,
"status": <string>,
"noop": <boolean>
}
**NOTE** `metrics` and `logs` may also take the value null.
All keys are mandatory unless otherwise noted, though values that are lists may be empty lists.
`"certname"` is the certname the report is associated with.
`"report"` is the environment associated to the node at the time of the report
`"environment"` is the environment associated to the node at the time of the report
`"puppet_version"` is the version of puppet that was run to generate this report.
......@@ -53,6 +58,46 @@ match a report with the catalog that was used for the run. This field may be `n
`"noop"` is a flag that indicates whether the report was produced with a --noop run.
`"resource_events"` is an array of objects of the following form:
{
"status": <status of event (`success`, `failure`, `noop`, or `skipped`)>,
"timestamp": <timestamp (from agent) at which event occurred>,
"resource_type": <type of resource event occurred on>,
"resource_title": <title of resource event occurred on>,
"property": <property/parameter of resource on which event occurred>,
"new_value": <new value for resource property>,
"old_value": <old value of resource property>,
"message": <description of what happened during event>,
"file": <manifest file containing resource definition>,
"line": <line in manifest file on which resource is defined>,
"containment_path": <containment heirarchy of resource within catalog>
}
`"metrics"` is either null or an array of metric objects like so:
{
"category" : <category of metric ("resources", "time", "changes", or "events")>,
"name" : <name of the metric>,
"value" : <value of the metric (double precision)>
}
`"logs"` is either null or an array of log objects like so:
{
"file" : <file of resource declaration>,
"line" : <line of resource declaration>,
"level" : <log level>,
"message" : <log message>,
"source" : <log source>,
"tags" : [<resource tag>],
"time" : <log line timestamp>
}
**Note on fields that allow `NULL` values**
In the resource_event schema above, `containment_path`, `new_value`, `old_value`, `property`, `file`, `line`, `status`, and `message` may all be null.
### Encoding
The entire report is expected to be valid JSON, which implies UTF-8
......
......@@ -52,6 +52,8 @@ Puppet::Reports.register_report(:puppetdb) do
"transaction_uuid" => transaction_uuid,
"status" => status,
"noop" => is_noop,
"logs" => build_logs_list,
"metrics" => build_metrics_list,
}
end
end
......@@ -73,6 +75,40 @@ Puppet::Reports.register_report(:puppetdb) do
end
end
# @return Array[Hash]
# @api private
def build_logs_list
profile("Build logs list (count: #{logs.count})",
[:puppetdb, :logs_list, :build]) do
logs.map do |log|
{
'file' => log.file,
'line' => log.line,
'level' => log.level,
'message' => log.message,
'source' => log.source,
'tags' => [*log.tags],
'time' => Puppet::Util::Puppetdb.to_wire_time(log.time),
}
end
end
end
# @return Array[Hash}
# @api private
def build_metrics_list
profile("Build metrics list (count: #{metrics.count})",
[:puppetdb, :metrics_list, :build]) do
metrics_list = []
metrics.each do |name, data|
metric_hashes = data.values.map {|x| {"category" => data.name, "name" => x.first, "value" => x.last}}
metrics_list.concat(metric_hashes)
end
metrics_list
end
end
# @return Number
# @api private
def run_duration
......
(ns puppetlabs.puppetdb.anonymizer
(:require [puppetlabs.puppetdb.reports :as report]
[puppetlabs.puppetdb.utils :as utils]
[schema.core :as s]
[puppetlabs.puppetdb.schema :as pls]
[clojure.string :as str]
[puppetlabs.kitchensink.core :refer [regexp? boolean? uuid string-contains?]]
......@@ -17,19 +18,13 @@
(contains? edge "target")
(contains? edge "relationship")))
(defn str-schema
"Function for converting a schema with keyword keys to
to one with string keys. Doens't walk the map so nested
schema won't work."
[kwd-schema]
(reduce-kv (fn [acc k v]
(assoc acc (schema.core/required-key (puppetlabs.puppetdb.utils/kwd->str k)) v))
{} kwd-schema))
(def resource-event-schema-str (str-schema report/resource-event-schema))
(def report-schema-str (-> report/report-schema
(assoc :resource_events [resource-event-schema-str])
str-schema))
(def resource-event-schema-str (utils/str-schema report/resource-event-schema))
(def metric-schema-str (utils/str-schema report/metric-schema))
(def log-schema-str (utils/str-schema report/log-schema))
(def report-schema-str (utils/str-schema (assoc report/report-schema
:resource_events [resource-event-schema-str]
:metrics (s/maybe [metric-schema-str])
:logs (s/maybe [log-schema-str]))))
(defn resource?
"Returns true if it looks like a resource"
......
(ns puppetlabs.puppetdb.query.reports
(:require [puppetlabs.puppetdb.query-eng.engine :as qe]
[puppetlabs.puppetdb.cheshire :as json]
[puppetlabs.puppetdb.reports :as reports]
[puppetlabs.puppetdb.scf.storage-utils :as scf-utils]
[puppetlabs.puppetdb.schema :as pls]
[puppetlabs.kitchensink.core :as kitchensink]
[puppetlabs.puppetdb.jdbc :as jdbc]
......@@ -9,7 +11,8 @@
[clojure.set :refer [rename-keys]]
[puppetlabs.puppetdb.query.paging :as paging]
[puppetlabs.puppetdb.query :as query]
[puppetlabs.puppetdb.utils :as utils]))
[puppetlabs.puppetdb.utils :as utils])
(:import [org.postgresql.util PGobject]))
(def row-schema
{:hash String
......@@ -17,6 +20,8 @@
:puppet_version String
:report_format s/Int
:configuration_version String
:metrics (s/maybe (s/either String PGobject))
:logs (s/maybe (s/either String PGobject))
:start_time pls/Timestamp
:end_time pls/Timestamp
:receive_time pls/Timestamp
......@@ -49,6 +54,12 @@
:status (s/maybe String)
:message (s/maybe String)})
(def json-metric-schema
(utils/str-schema reports/metric-schema))
(def json-log-schema
(utils/str-schema reports/log-schema))
(def report-schema
{:hash String
(s/optional-key :environment) (s/maybe String)
......@@ -61,6 +72,8 @@
:report_format s/Int
:configuration_version String
:resource_events [resource-event-schema]
:metrics (s/maybe [json-metric-schema])
:logs (s/maybe [json-log-schema])
:transaction_uuid String
:status (s/maybe String)})
......@@ -76,6 +89,8 @@
:status
:environment
:configuration_version
:metrics
:logs
:certname])
(defn create-report-pred
......@@ -92,8 +107,7 @@
:message])]
(into acc
[(-> resource-event
(update-in [:new_value] json/parse-string)
(update-in [:old_value] json/parse-string)
((partial kitchensink/maptrans {[:new_value :old_value] json/parse-string}))
(rename-keys {:event_status :status}))])))
(pls/defn-validated collapse-report :- report-schema
......@@ -102,8 +116,9 @@
(let [first-row (first report-rows)
resource-events (->> report-rows
(reduce collapse-resource-events []))]
(assoc (select-keys first-row report-columns)
:resource_events resource-events)))
(-> (select-keys first-row report-columns)
((partial kitchensink/maptrans {[:metrics :logs] (scf-utils/parse-db-json-fn)}))
(assoc :resource_events resource-events))))
(pls/defn-validated structured-data-seq
"Produce a lazy seq of catalogs from a list of rows ordered by catalog hash"
......
......@@ -175,6 +175,8 @@
"puppet_version" :string
"report_format" :number
"configuration_version" :string
"metrics" :string
"logs" :string
"old_value" :string
"new_value" :string
"timestamp" :timestamp
......@@ -212,6 +214,8 @@
reports.receive_time,
reports.transaction_uuid,
reports.noop,
reports.metrics,
reports.logs,
environments.name as environment,
report_statuses.status as status,
re.report_id,
......
......@@ -19,6 +19,20 @@
:line (s/maybe s/Int)
:containment_path [s/Str]})
(def metric-schema
{:category s/Str
:name s/Str
:value s/Num})
(def log-schema
{:file (s/maybe s/Str)
:line (s/maybe s/Int)
:level s/Str
:message s/Str
:source s/Str
:tags [s/Str]
:time pls/Timestamp})
(def report-schema
{:certname s/Str
:puppet_version s/Str
......@@ -29,6 +43,8 @@
:resource_events [resource-event-schema]
:noop (s/maybe s/Bool)
:transaction_uuid (s/maybe s/Str)
:metrics (s/maybe [metric-schema])
:logs (s/maybe [log-schema])
:environment s/Str
:status s/Str})
......
......@@ -1021,6 +1021,18 @@
"ALTER TABLE certnames RENAME COLUMN name TO certname"
"ALTER TABLE certnames ALTER COLUMN name RENAME TO certname")))
(defn insert-report-metrics-and-logs
"Insert columns in reports to be populated by metrics and logs.
Text for hsql, JSON for postgres."
[]
(if (scf-utils/postgres?)
(sql/do-commands
"ALTER TABLE reports ADD metrics json"
"ALTER TABLE reports ADD logs json")
(sql/do-commands
"ALTER TABLE reports ADD metrics text"
"ALTER TABLE reports ADD logs text")))
(def migrations
"The available migrations, as a map from migration version to migration function."
{1 initialize-store
......@@ -1052,7 +1064,8 @@
27 switch-value-string-index-to-gin
28 insert-factset-hash-column
29 migrate-to-report-id-and-noop-column-and-drop-latest-reports
30 change-name-to-certname})
30 change-name-to-certname
31 insert-report-metrics-and-logs})
(def desired-schema-version (apply max (keys migrations)))
......
This diff is collapsed.
......@@ -4,7 +4,8 @@
[puppetlabs.puppetdb.jdbc :as jdbc]
[puppetlabs.kitchensink.core :as kitchensink]
[puppetlabs.puppetdb.schema :as pls]
[schema.core :as s]))
[schema.core :as s])
(:import [org.postgresql.util PGobject]))
;; SCHEMA
......@@ -252,3 +253,30 @@ must be supplied as the value to be matched."
(sql/do-commands
(str "ALTER TABLE " table " ALTER COLUMN " column
" RESTART WITH " restartid))))))
(pls/defn-validated clj->pgjson :- PGobject
"Convert a clojure object to a json PGobject"
[value :- (s/maybe (s/either {s/Any s/Any} [{s/Any s/Any}]))]
(doto (PGobject.)
(.setType "json")
(.setValue (json/generate-string value))))
(pls/defn-validated pgjson->clj
"Convert a json PGobject to a clojure object"
[value :- (s/maybe PGobject)]
(when value
(json/parse-string (.getValue value))))
(defn munge-json-for-storage
"Prepare a clojure object for storage depending on db type."
[value]
(if (postgres?)
(clj->pgjson value)
(json/generate-string value)))
(defn parse-db-json-fn
"Produce a function for parsing an object stored as json."
[]
(if (postgres?)
pgjson->clj
json/parse-string))
......@@ -200,3 +200,12 @@
(let [certname (:certname (first rows))]
(fn [row]
(= certname (:certname row)))))
(defn str-schema
"Function for converting a schema with keyword keys to
to one with string keys. Doens't walk the map so nested
schema won't work."
[kwd-schema]
(reduce-kv (fn [acc k v]
(assoc acc (schema.core/required-key (puppetlabs.puppetdb.utils/kwd->str k)) v))
{} kwd-schema))
......@@ -17,6 +17,8 @@
"new_value": "\"Hi world\"",
"old_value": "\"absent\"",
"property": "message",
"metrics" : null,
"logs" : null,
"puppet_version": "3.7.3",
"receive_time": "2014-12-24T00:00:50Z",
"report_format": 4,
......@@ -49,6 +51,8 @@
"puppet_version": "3.7.3",
"receive_time": "2014-12-24T00:00:50Z",
"report_format": 4,
"metrics" : null,
"logs" : null,
"resource_title": "/home/wyatt/Desktop/foo",
"resource_type": "File",
"start_time": "2014-12-24T00:00:49Z",
......@@ -78,6 +82,8 @@
"puppet_version": "3.7.3",
"receive_time": "2014-12-24T00:01:12Z",
"report_format": 4,
"metrics" : null,
"logs" : null,
"resource_title": "hi",
"resource_type": "Notify",
"start_time": "2014-12-24T00:01:11Z",
......
......@@ -38,5 +38,41 @@
"transaction_uuid": "68b08e2a-eeb1-4322-b241-bfdf151d294b",
"environment": "DEV",
"status" : "unchanged",
"metrics": [
{
"category": "resources",
"name": "changed",
"value": 1.1
},
{
"category": "resources",
"name": "failed",
"value": 3.14
},
{
"category": "resources",
"name": "failed_to_restart",
"value": -70000000.0
}
],
"logs" : [
{
"file" : null,
"line" : null,
"level" : "info",
"message" : "Caching catalog for mbp.local",
"source" : "//mbp.local/Puppet",
"tags" : [ "info" ],
"time" : "2015-02-26T15:20:17.321565000-08:00"
}, {
"file" : null,
"line" : null,