Commit d15925a1 authored by Andrew Roetker's avatar Andrew Roetker

(PDB-1984) Migrate benchmarking tool to submit v6 reports

This commit migrates the benchmarking tool to "sanitize" its example v5
wireformat reports to v6 reports, and removes some unnecessary json
generating and unparsing we do in the tool.
Since we're removing the json generating and parsing for reports, the
same unnecessary json roundtrips are removed from catalogs and facts as
well for consistency.
parent ab4916b9
......@@ -47,6 +47,7 @@
[puppetlabs.kitchensink.core :as kitchensink]
[clj-time.core :as time]
[puppetlabs.puppetdb.client :as client]
[puppetlabs.puppetdb.reports :as reports]
[puppetlabs.puppetdb.random :refer [random-string random-bool]]
[puppetlabs.puppetdb.archive :as archive]
[slingshot.slingshot :refer [try+ throw+]]
......@@ -150,7 +151,9 @@
"transaction_uuid" uuid
"start_time" (time/minus stamp (time/seconds 10))
"end_time" (time/minus stamp (time/seconds 5))
"producer_timestamp" stamp)))
"producer_timestamp" stamp)
clojure.walk/keywordize-keys
reports/sanitize-report))
(defn randomize-map-leaf
"Randomizes a fact leaf."
......@@ -198,9 +201,9 @@
(update-report uuid stamp))
factset (some-> factset
(update-factset rand-percentage stamp))]
(when catalog (>!! command-send-ch [:catalog 6 (json/generate-string catalog)]))
(when report (>!! command-send-ch [:report 5 (json/generate-string report)]))
(when factset (>!! command-send-ch [:factset 4 (json/generate-string factset)]))
(when catalog (>!! command-send-ch [:catalog 6 catalog]))
(when report (>!! command-send-ch [:report 5 report]))
(when factset (>!! command-send-ch [:factset 4 factset]))
(assoc state
:catalog catalog
......
......@@ -9,7 +9,6 @@
[puppetlabs.puppetdb.schema :refer [defn-validated]]
[puppetlabs.puppetdb.utils :as utils]
[puppetlabs.kitchensink.core :as kitchensink]
[clojure.walk :refer [keywordize-keys]]
[schema.core :as s]))
(defn-validated submit-command-via-http!
......@@ -58,12 +57,11 @@
command-processing endpoint located at `puppetdb-host`:`puppetdb-port`."
[base-url :- utils/base-url-schema
command-version :- s/Int
catalog-payload :- s/Str]
(let [payload (json/parse-string catalog-payload)
result (submit-command-via-http!
base-url
(command-names :replace-catalog) command-version
payload)]
catalog-payload]
(let [result (submit-command-via-http!
base-url
(command-names :replace-catalog) command-version
catalog-payload)]
(when-not (= http/status-ok (:status result))
(log/error result))))
......@@ -72,15 +70,11 @@
command-processing endpoint located at `puppetdb-host`:`puppetdb-port`."
[base-url :- utils/base-url-schema
command-version :- s/Int
report-payload :- s/Str]
(let [payload (-> report-payload
json/parse-string
keywordize-keys
reports/sanitize-report)
result (submit-command-via-http!
base-url
(command-names :store-report) command-version
payload)]
report-payload]
(let [result (submit-command-via-http!
base-url
(command-names :store-report) command-version
report-payload)]
(when-not (= http/status-ok (:status result))
(log/error result))))
......@@ -89,14 +83,10 @@
command-processing endpoint located at `puppetdb-host`:`puppetdb-port`."
[base-url :- utils/base-url-schema
facts-version :- s/Int
fact-payload :- s/Str]
(let [payload (case facts-version
1 fact-payload
(json/parse-string fact-payload))
result (submit-command-via-http!
fact-payload]
(let [result (submit-command-via-http!
base-url
(command-names :replace-facts)
facts-version
payload)]
(command-names :replace-facts) facts-version
fact-payload)]
(when-not (= http/status-ok (:status result))
(log/error result))))
......@@ -80,20 +80,6 @@
(def report-v3-wireformat-schema
(dissoc report-v4-wireformat-schema :status))
(pls/defn-validated sanitize-v5-resource-events :- [resource-event-v5-wireformat-schema]
"This function takes an array of events and santizes them, ensuring only
valid keys are returned."
[resource-events :- (s/pred coll? 'coll?)]
(for [resource-event resource-events]
(pls/strip-unknown-keys resource-event-v5-wireformat-schema resource-event)))
(pls/defn-validated sanitize-report :- report-v5-wireformat-schema
"This function takes a report and sanitizes it, ensuring only valid data
is left over."
[payload :- (s/pred map? 'map?)]
(-> (pls/strip-unknown-keys report-v5-wireformat-schema payload)
(update :resource_events sanitize-v5-resource-events)))
(def resource-event-query-schema
{(s/optional-key :certname) s/Str
(s/optional-key :report_receive_time) pls/Timestamp
......@@ -177,16 +163,6 @@
(defn reports-query->wire-v5 [reports]
(map report-query->wire-v5 reports))
(declare wire-v5->wire-v6)
(pls/defn-validated report-query->wire-v6 :- report-wireformat-schema
[report :- report-query-schema]
(-> report
report-query->wire-v5
wire-v5->wire-v6))
(defn reports-query->wire-v6 [reports]
(map report-query->wire-v6 reports))
(defn dash->underscore-report-keys [v5-report-or-older]
(->> v5-report-or-older
utils/dash->underscore-keys
......@@ -229,6 +205,35 @@
(assoc :status nil)
(wire-v4->wire-v6 received-time)))
(pls/defn-validated report-query->wire-v6 :- report-wireformat-schema
[report :- report-query-schema]
(-> report
report-query->wire-v5
wire-v5->wire-v6))
(defn reports-query->wire-v6 [reports]
(map report-query->wire-v6 reports))
(pls/defn-validated sanitize-events :- [event-wireformat-schema]
[events]
(for [event events]
(pls/strip-unknown-keys event-wireformat-schema event)))
(pls/defn-validated sanitize-resources :- [resource-wireformat-schema]
[resources]
(for [resource resources]
(-> (pls/strip-unknown-keys resource-wireformat-schema resource)
(update :events sanitize-events))))
(pls/defn-validated sanitize-report :- report-wireformat-schema
"This function takes a report and sanitizes it, ensuring only valid data
is left over."
[payload]
(as-> payload $
(wire-v5->wire-v6 $)
(pls/strip-unknown-keys report-wireformat-schema $)
(update $ :resources sanitize-resources)))
(defn- resource->skipped-resource-events
"Fabricate a skipped resource-event"
[resource]
......
(ns puppetlabs.puppetdb.cli.benchmark-test
(:require [puppetlabs.puppetdb.cli.benchmark :as benchmark]
[clojure.test :refer :all]
[puppetlabs.puppetdb.cheshire :as json]
[puppetlabs.puppetdb.archive :as archive]
[puppetlabs.puppetdb.client :as client]
[puppetlabs.trapperkeeper.config :as config]
......@@ -18,7 +17,7 @@
:base-url base-url
:version version
:payload-string payload-string
:payload (json/parse-string payload-string true)})))
:payload (clojure.walk/keywordize-keys payload-string)})))
(defn run-benchmark [config & cli-args]
(let [submitted-records (atom [])]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment