Commit 9cd6d8a7 authored by Scott Moser's avatar Scott Moser

sync curtin reporting changes back to cloud-init

this pulls in the reporting framework that we'd been working on
in curtin back to cloud-init.
parents deb6cffa cba8282a
......@@ -24,7 +24,7 @@ import copy
import os
from cloudinit import log as logging
from cloudinit import reporting
from cloudinit.reporting import events
LOG = logging.getLogger(__name__)
......@@ -48,7 +48,7 @@ class Cloud(object):
self._cfg = cfg
self._runners = runners
if reporter is None:
reporter = reporting.ReportEventStack(
reporter = events.ReportEventStack(
name="unnamed-cloud-reporter",
description="unnamed-cloud-reporter",
reporting_enabled=False)
......
# Copyright 2015 Canonical Ltd.
# This file is part of cloud-init. See LICENCE file for license information.
#
# vi: ts=4 expandtab
"""
cloud-init reporting framework
......@@ -10,66 +9,13 @@ report events in a structured manner.
"""
from ..registry import DictRegistry
from ..reporting.handlers import available_handlers
FINISH_EVENT_TYPE = 'finish'
START_EVENT_TYPE = 'start'
from .handlers import available_handlers
DEFAULT_CONFIG = {
'logging': {'type': 'log'},
}
class _nameset(set):
def __getattr__(self, name):
if name in self:
return name
raise AttributeError("%s not a valid value" % name)
status = _nameset(("SUCCESS", "WARN", "FAIL"))
class ReportingEvent(object):
"""Encapsulation of event formatting."""
def __init__(self, event_type, name, description):
self.event_type = event_type
self.name = name
self.description = description
def as_string(self):
"""The event represented as a string."""
return '{0}: {1}: {2}'.format(
self.event_type, self.name, self.description)
def as_dict(self):
"""The event represented as a dictionary."""
return {'name': self.name, 'description': self.description,
'event_type': self.event_type}
class FinishReportingEvent(ReportingEvent):
def __init__(self, name, description, result=status.SUCCESS):
super(FinishReportingEvent, self).__init__(
FINISH_EVENT_TYPE, name, description)
self.result = result
if result not in status:
raise ValueError("Invalid result: %s" % result)
def as_string(self):
return '{0}: {1}: {2}: {3}'.format(
self.event_type, self.name, self.result, self.description)
def as_dict(self):
"""The event represented as json friendly."""
data = super(FinishReportingEvent, self).as_dict()
data['result'] = self.result
return data
def update_configuration(config):
"""Update the instanciated_handler_registry.
......@@ -90,150 +36,7 @@ def update_configuration(config):
instantiated_handler_registry.register_item(handler_name, instance)
def report_event(event):
"""Report an event to all registered event handlers.
This should generally be called via one of the other functions in
the reporting module.
:param event_type:
The type of the event; this should be a constant from the
reporting module.
"""
for _, handler in instantiated_handler_registry.registered_items.items():
handler.publish_event(event)
def report_finish_event(event_name, event_description,
result=status.SUCCESS):
"""Report a "finish" event.
See :py:func:`.report_event` for parameter details.
"""
event = FinishReportingEvent(event_name, event_description, result)
return report_event(event)
def report_start_event(event_name, event_description):
"""Report a "start" event.
:param event_name:
The name of the event; this should be a topic which events would
share (e.g. it will be the same for start and finish events).
:param event_description:
A human-readable description of the event that has occurred.
"""
event = ReportingEvent(START_EVENT_TYPE, event_name, event_description)
return report_event(event)
class ReportEventStack(object):
"""Context Manager for using :py:func:`report_event`
This enables calling :py:func:`report_start_event` and
:py:func:`report_finish_event` through a context manager.
:param name:
the name of the event
:param description:
the event's description, passed on to :py:func:`report_start_event`
:param message:
the description to use for the finish event. defaults to
:param:description.
:param parent:
:type parent: :py:class:ReportEventStack or None
The parent of this event. The parent is populated with
results of all its children. The name used in reporting
is <parent.name>/<name>
:param reporting_enabled:
Indicates if reporting events should be generated.
If not provided, defaults to the parent's value, or True if no parent
is provided.
:param result_on_exception:
The result value to set if an exception is caught. default
value is FAIL.
"""
def __init__(self, name, description, message=None, parent=None,
reporting_enabled=None, result_on_exception=status.FAIL):
self.parent = parent
self.name = name
self.description = description
self.message = message
self.result_on_exception = result_on_exception
self.result = status.SUCCESS
# use parents reporting value if not provided
if reporting_enabled is None:
if parent:
reporting_enabled = parent.reporting_enabled
else:
reporting_enabled = True
self.reporting_enabled = reporting_enabled
if parent:
self.fullname = '/'.join((parent.fullname, name,))
else:
self.fullname = self.name
self.children = {}
def __repr__(self):
return ("ReportEventStack(%s, %s, reporting_enabled=%s)" %
(self.name, self.description, self.reporting_enabled))
def __enter__(self):
self.result = status.SUCCESS
if self.reporting_enabled:
report_start_event(self.fullname, self.description)
if self.parent:
self.parent.children[self.name] = (None, None)
return self
def _childrens_finish_info(self):
for cand_result in (status.FAIL, status.WARN):
for name, (value, msg) in self.children.items():
if value == cand_result:
return (value, self.message)
return (self.result, self.message)
@property
def result(self):
return self._result
@result.setter
def result(self, value):
if value not in status:
raise ValueError("'%s' not a valid result" % value)
self._result = value
@property
def message(self):
if self._message is not None:
return self._message
return self.description
@message.setter
def message(self, value):
self._message = value
def _finish_info(self, exc):
# return tuple of description, and value
if exc:
return (self.result_on_exception, self.message)
return self._childrens_finish_info()
def __exit__(self, exc_type, exc_value, traceback):
(result, msg) = self._finish_info(exc_value)
if self.parent:
self.parent.children[self.name] = (result, msg)
if self.reporting_enabled:
report_finish_event(self.fullname, msg, result)
instantiated_handler_registry = DictRegistry()
update_configuration(DEFAULT_CONFIG)
# vi: ts=4 expandtab
# Copyright 2015 Canonical Ltd.
# This file is part of cloud-init. See LICENCE file for license information.
#
"""
events for reporting.
The events here are designed to be used with reporting.
They can be published to registered handlers with report_event.
"""
import base64
import os.path
import time
from . import instantiated_handler_registry
FINISH_EVENT_TYPE = 'finish'
START_EVENT_TYPE = 'start'
DEFAULT_EVENT_ORIGIN = 'cloudinit'
class _nameset(set):
def __getattr__(self, name):
if name in self:
return name
raise AttributeError("%s not a valid value" % name)
status = _nameset(("SUCCESS", "WARN", "FAIL"))
class ReportingEvent(object):
"""Encapsulation of event formatting."""
def __init__(self, event_type, name, description,
origin=DEFAULT_EVENT_ORIGIN, timestamp=time.time()):
self.event_type = event_type
self.name = name
self.description = description
self.origin = origin
self.timestamp = timestamp
def as_string(self):
"""The event represented as a string."""
return '{0}: {1}: {2}'.format(
self.event_type, self.name, self.description)
def as_dict(self):
"""The event represented as a dictionary."""
return {'name': self.name, 'description': self.description,
'event_type': self.event_type, 'origin': self.origin,
'timestamp': self.timestamp}
class FinishReportingEvent(ReportingEvent):
def __init__(self, name, description, result=status.SUCCESS,
post_files=None):
super(FinishReportingEvent, self).__init__(
FINISH_EVENT_TYPE, name, description)
self.result = result
if post_files is None:
post_files = []
self.post_files = post_files
if result not in status:
raise ValueError("Invalid result: %s" % result)
def as_string(self):
return '{0}: {1}: {2}: {3}'.format(
self.event_type, self.name, self.result, self.description)
def as_dict(self):
"""The event represented as json friendly."""
data = super(FinishReportingEvent, self).as_dict()
data['result'] = self.result
if self.post_files:
data['files'] = _collect_file_info(self.post_files)
return data
def report_event(event):
"""Report an event to all registered event handlers.
This should generally be called via one of the other functions in
the reporting module.
:param event_type:
The type of the event; this should be a constant from the
reporting module.
"""
for _, handler in instantiated_handler_registry.registered_items.items():
handler.publish_event(event)
def report_finish_event(event_name, event_description,
result=status.SUCCESS, post_files=None):
"""Report a "finish" event.
See :py:func:`.report_event` for parameter details.
"""
event = FinishReportingEvent(event_name, event_description, result,
post_files=post_files)
return report_event(event)
def report_start_event(event_name, event_description):
"""Report a "start" event.
:param event_name:
The name of the event; this should be a topic which events would
share (e.g. it will be the same for start and finish events).
:param event_description:
A human-readable description of the event that has occurred.
"""
event = ReportingEvent(START_EVENT_TYPE, event_name, event_description)
return report_event(event)
class ReportEventStack(object):
"""Context Manager for using :py:func:`report_event`
This enables calling :py:func:`report_start_event` and
:py:func:`report_finish_event` through a context manager.
:param name:
the name of the event
:param description:
the event's description, passed on to :py:func:`report_start_event`
:param message:
the description to use for the finish event. defaults to
:param:description.
:param parent:
:type parent: :py:class:ReportEventStack or None
The parent of this event. The parent is populated with
results of all its children. The name used in reporting
is <parent.name>/<name>
:param reporting_enabled:
Indicates if reporting events should be generated.
If not provided, defaults to the parent's value, or True if no parent
is provided.
:param result_on_exception:
The result value to set if an exception is caught. default
value is FAIL.
"""
def __init__(self, name, description, message=None, parent=None,
reporting_enabled=None, result_on_exception=status.FAIL,
post_files=None):
self.parent = parent
self.name = name
self.description = description
self.message = message
self.result_on_exception = result_on_exception
self.result = status.SUCCESS
if post_files is None:
post_files = []
self.post_files = post_files
# use parents reporting value if not provided
if reporting_enabled is None:
if parent:
reporting_enabled = parent.reporting_enabled
else:
reporting_enabled = True
self.reporting_enabled = reporting_enabled
if parent:
self.fullname = '/'.join((parent.fullname, name,))
else:
self.fullname = self.name
self.children = {}
def __repr__(self):
return ("ReportEventStack(%s, %s, reporting_enabled=%s)" %
(self.name, self.description, self.reporting_enabled))
def __enter__(self):
self.result = status.SUCCESS
if self.reporting_enabled:
report_start_event(self.fullname, self.description)
if self.parent:
self.parent.children[self.name] = (None, None)
return self
def _childrens_finish_info(self):
for cand_result in (status.FAIL, status.WARN):
for name, (value, msg) in self.children.items():
if value == cand_result:
return (value, self.message)
return (self.result, self.message)
@property
def result(self):
return self._result
@result.setter
def result(self, value):
if value not in status:
raise ValueError("'%s' not a valid result" % value)
self._result = value
@property
def message(self):
if self._message is not None:
return self._message
return self.description
@message.setter
def message(self, value):
self._message = value
def _finish_info(self, exc):
# return tuple of description, and value
if exc:
return (self.result_on_exception, self.message)
return self._childrens_finish_info()
def __exit__(self, exc_type, exc_value, traceback):
(result, msg) = self._finish_info(exc_value)
if self.parent:
self.parent.children[self.name] = (result, msg)
if self.reporting_enabled:
report_finish_event(self.fullname, msg, result,
post_files=self.post_files)
def _collect_file_info(files):
if not files:
return None
ret = []
for fname in files:
if not os.path.isfile(fname):
content = None
else:
with open(fname, "rb") as fp:
content = base64.b64encode(fp.read()).decode()
ret.append({'path': fname, 'content': content,
'encoding': 'base64'})
return ret
# vi: ts=4 expandtab syntax=python
# vi: ts=4 expandtab
import abc
import oauthlib.oauth1 as oauth1
import six
from ..registry import DictRegistry
......@@ -22,12 +21,11 @@ class ReportingHandler(object):
@abc.abstractmethod
def publish_event(self, event):
"""Publish an event to the ``INFO`` log level."""
print(event.as_string())
"""Publish an event."""
class LogHandler(ReportingHandler):
"""Publishes events to the cloud-init log at the ``INFO`` log level."""
"""Publishes events to the cloud-init log at the ``DEBUG`` log level."""
def __init__(self, level="DEBUG"):
super(LogHandler, self).__init__()
......@@ -43,15 +41,16 @@ class LogHandler(ReportingHandler):
self.level = level
def publish_event(self, event):
"""Publish an event to the ``INFO`` log level."""
logger = logging.getLogger(
'.'.join(['cloudinit', 'reporting', event.event_type, event.name]))
logger.log(self.level, event.as_string())
class PrintHandler(ReportingHandler):
"""Print the event as a string."""
def publish_event(self, event):
"""Publish an event to the ``INFO`` log level."""
print(event.as_string())
class WebHookHandler(ReportingHandler):
......
......@@ -27,12 +27,12 @@ import six
from cloudinit import importer
from cloudinit import log as logging
from cloudinit import reporting
from cloudinit import type_utils
from cloudinit import user_data as ud
from cloudinit import util
from cloudinit.filters import launch_index
from cloudinit.reporting import events
DEP_FILESYSTEM = "FILESYSTEM"
DEP_NETWORK = "NETWORK"
......@@ -254,7 +254,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
LOG.debug("Searching for %s data source in: %s", mode, ds_names)
for name, cls in zip(ds_names, ds_list):
myrep = reporting.ReportEventStack(
myrep = events.ReportEventStack(
name="search-%s" % name.replace("DataSource", ""),
description="searching for %s data from %s" % (mode, name),
message="no %s data found from %s" % (mode, name),
......
......@@ -46,7 +46,7 @@ from cloudinit import log as logging
from cloudinit import sources
from cloudinit import type_utils
from cloudinit import util
from cloudinit import reporting
from cloudinit.reporting import events
LOG = logging.getLogger(__name__)
......@@ -67,7 +67,7 @@ class Init(object):
self.datasource = NULL_DATA_SOURCE
if reporter is None:
reporter = reporting.ReportEventStack(
reporter = events.ReportEventStack(
name="init-reporter", description="init-desc",
reporting_enabled=False)
self.reporter = reporter
......@@ -242,7 +242,7 @@ class Init(object):
if self.datasource is not NULL_DATA_SOURCE:
return self.datasource
with reporting.ReportEventStack(
with events.ReportEventStack(
name="check-cache",
description="attempting to read from cache",
parent=self.reporter) as myrep:
......@@ -509,11 +509,11 @@ class Init(object):
def consume_data(self, frequency=PER_INSTANCE):
# Consume the userdata first, because we need want to let the part
# handlers run first (for merging stuff)
with reporting.ReportEventStack(
with events.ReportEventStack(
"consume-user-data", "reading and applying user-data",
parent=self.reporter):
self._consume_userdata(frequency)
with reporting.ReportEventStack(
with events.ReportEventStack(
"consume-vendor-data", "reading and applying vendor-data",
parent=self.reporter):
self._consume_vendordata(frequency)
......@@ -595,7 +595,7 @@ class Modules(object):
# Created on first use
self._cached_cfg = None
if reporter is None:
reporter = reporting.ReportEventStack(
reporter = events.ReportEventStack(
name="module-reporter", description="module-desc",
reporting_enabled=False)
self.reporter = reporter
......@@ -710,7 +710,7 @@ class Modules(object):
run_name = "config-%s" % (name)
desc = "running %s with frequency %s" % (run_name, freq)
myrep = reporting.ReportEventStack(
myrep = events.ReportEventStack(
name=run_name, description=desc, parent=self.reporter)
with myrep:
......
......@@ -211,10 +211,14 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
manual_tries = 1
if retries:
manual_tries = max(int(retries) + 1, 1)
if not headers:
headers = {
'User-Agent': 'Cloud-Init/%s' % (version.version_string()),
}
def_headers = {
'User-Agent': 'Cloud-Init/%s' % (version.version_string()),
}
if headers:
def_headers.update(headers)
headers = def_headers
if not headers_cb:
def _cb(url):
return headers
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment