Commit d6799be5 authored by Thomas Goirand's avatar Thomas Goirand

Merge tag 'upstream/18.3'

Upstream version 18.3
parents 0ab4730b 144eedf4
......@@ -5,3 +5,9 @@ dist
__pycache__
.tox
.coverage
doc/rtd_html
parts
prime
stage
*.snap
*.cover
[MASTER]
# --go-faster, use multiple processes to speed up Pylint
jobs=4
[MESSAGES CONTROL]
# Errors and warings with some filtered:
# W0105(pointless-string-statement)
# W0107(unnecessary-pass)
# W0201(attribute-defined-outside-init)
# W0212(protected-access)
# W0221(arguments-differ)
# W0222(signature-differs)
# W0223(abstract-method)
# W0231(super-init-not-called)
# W0311(bad-indentation)
# W0511(fixme)
# W0602(global-variable-not-assigned)
# W0603(global-statement)
# W0611(unused-import)
# W0612(unused-variable)
# W0613(unused-argument)
# W0621(redefined-outer-name)
# W0622(redefined-builtin)
# W0631(undefined-loop-variable)
# W0703(broad-except)
# W1401(anomalous-backslash-in-string)
disable=C, F, I, R, W0105, W0107, W0201, W0212, W0221, W0222, W0223, W0231, W0311, W0511, W0602, W0603, W0611, W0613, W0621, W0622, W0631, W0703, W1401
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
output-format=parseable
# Just the errors please, no full report
reports=no
[TYPECHECK]
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
http.client,
httplib,
pkg_resources,
six.moves,
# cloud_tests requirements.
boto3,
botocore,
paramiko,
pylxd,
simplestreams
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=types,http.client,command_handlers,m_.*
This diff is collapsed.
......@@ -13,6 +13,17 @@ Do these things once
If you have already signed it as an individual, your Launchpad user will be listed in the `contributor-agreement-canonical`_ group. Unfortunately there is no easy way to check if an organization or company you are doing work for has signed. If you are unsure or have questions, email `Scott Moser <mailto:scott.moser@canonical.com>`_ or ping smoser in ``#cloud-init`` channel via freenode.
When prompted for 'Project contact' or 'Canonical Project Manager' enter
'Scott Moser'.
* Configure git with your email and name for commit messages.
Your name will appear in commit messages and will also be used in
changelogs or release notes. Give yourself credit!::
git config user.name "Your Name"
git config user.email "Your Email"
* Clone the upstream `repository`_ on Launchpad::
git clone https://git.launchpad.net/cloud-init
......
include *.py MANIFEST.in ChangeLog
include *.py MANIFEST.in LICENSE* ChangeLog
global-include *.txt *.rst *.ini *.in *.conf *.cfg *.sh
graft bash_completion
graft config
graft doc
graft packages
graft systemd
graft sysvinit
graft templates
graft tests
graft tools
graft udev
graft upstart
prune build
prune dist
prune .tox
......
CWD=$(shell pwd)
PYVER ?= $(shell for p in python3 python2; do \
out=$(which $$p 2>&1) && echo $$p && exit; done; \
exit 1)
out=$$(command -v $$p 2>&1) && echo $$p && exit; done; exit 1)
noseopts ?= -v
YAML_FILES=$(shell find cloudinit bin tests tools -name "*.yaml" -type f )
YAML_FILES=$(shell find cloudinit tests tools -name "*.yaml" -type f )
YAML_FILES+=$(shell find doc/examples -name "cloud-config*.txt" -type f )
PIP_INSTALL := pip install
......@@ -27,13 +27,16 @@ ifeq ($(distro),)
distro = redhat
endif
READ_VERSION=$(shell $(PYVER) $(CWD)/tools/read-version)
READ_VERSION=$(shell $(PYVER) $(CWD)/tools/read-version || \
echo read-version-failed)
CODE_VERSION=$(shell $(PYVER) -c "from cloudinit import version; print(version.version_string())")
all: check
check: check_version pep8 $(pyflakes) test $(yaml)
check: check_version test $(yaml)
style-check: pep8 $(pyflakes)
pep8:
@$(CWD)/tools/run-pep8
......@@ -43,12 +46,18 @@ pyflakes:
pyflakes3:
@$(CWD)/tools/run-pyflakes3
unittest: clean_pyc
nosetests $(noseopts) tests/unittests
nosetests $(noseopts) tests/unittests cloudinit
unittest3: clean_pyc
nosetests3 $(noseopts) tests/unittests
nosetests3 $(noseopts) tests/unittests cloudinit
ci-deps-ubuntu:
@$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --test-distro
ci-deps-centos:
@$(PYVER) $(CWD)/tools/read-dependencies --distro centos --test-distro
pip-requirements:
@echo "Installing cloud-init dependencies..."
......@@ -62,10 +71,13 @@ test: $(unittests)
check_version:
@if [ "$(READ_VERSION)" != "$(CODE_VERSION)" ]; then \
echo "Error: read-version version $(READ_VERSION)" \
"not equal to code version $(CODE_VERSION)"; exit 2; \
echo "Error: read-version version '$(READ_VERSION)'" \
"not equal to code version '$(CODE_VERSION)'"; exit 2; \
else true; fi
config/cloud.cfg:
$(PYVER) ./tools/render-cloudcfg config/cloud.cfg.tmpl config/cloud.cfg
clean_pyc:
@find . -type f -name "*.pyc" -delete
......@@ -73,13 +85,28 @@ clean: clean_pyc
rm -rf /var/log/cloud-init.log /var/lib/cloud/
yaml:
@$(CWD)/tools/validate-yaml.py $(YAML_FILES)
@$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES)
rpm:
./packages/brpm --distro $(distro)
$(PYVER) ./packages/brpm --distro=$(distro)
srpm:
$(PYVER) ./packages/brpm --srpm --distro=$(distro)
deb:
./packages/bddeb
@which debuild || \
{ echo "Missing devscripts dependency. Install with:"; \
echo sudo apt-get install devscripts; exit 1; }
$(PYVER) ./packages/bddeb
deb-src:
@which debuild || \
{ echo "Missing devscripts dependency. Install with:"; \
echo sudo apt-get install devscripts; exit 1; }
$(PYVER) ./packages/bddeb -S -d
.PHONY: test pyflakes pyflakes3 clean pep8 rpm deb yaml check_version
.PHONY: pip-test-requirements pip-requirements clean_pyc unittest unittest3
.PHONY: test pyflakes pyflakes3 clean pep8 rpm srpm deb deb-src yaml
.PHONY: check_version pip-test-requirements pip-requirements clean_pyc
.PHONY: unittest unittest3 style-check
# Copyright (C) 2018 Canonical Ltd.
#
# This file is part of cloud-init. See LICENSE file for license information.
# bash completion for cloud-init cli
_cloudinit_complete()
{
local cur_word prev_word
cur_word="${COMP_WORDS[COMP_CWORD]}"
prev_word="${COMP_WORDS[COMP_CWORD-1]}"
subcmds="analyze clean collect-logs devel dhclient-hook features init modules single status"
base_params="--help --file --version --debug --force"
case ${COMP_CWORD} in
1)
COMPREPLY=($(compgen -W "$base_params $subcmds" -- $cur_word))
;;
2)
case ${prev_word} in
analyze)
COMPREPLY=($(compgen -W "--help blame dump show" -- $cur_word))
;;
clean)
COMPREPLY=($(compgen -W "--help --logs --reboot --seed" -- $cur_word))
;;
collect-logs)
COMPREPLY=($(compgen -W "--help --tarfile --include-userdata" -- $cur_word))
;;
devel)
COMPREPLY=($(compgen -W "--help schema" -- $cur_word))
;;
dhclient-hook|features)
COMPREPLY=($(compgen -W "--help" -- $cur_word))
;;
init)
COMPREPLY=($(compgen -W "--help --local" -- $cur_word))
;;
modules)
COMPREPLY=($(compgen -W "--help --mode" -- $cur_word))
;;
single)
COMPREPLY=($(compgen -W "--help --name --frequency --report" -- $cur_word))
;;
status)
COMPREPLY=($(compgen -W "--help --long --wait" -- $cur_word))
;;
esac
;;
3)
case ${prev_word} in
blame|dump)
COMPREPLY=($(compgen -W "--help --infile --outfile" -- $cur_word))
;;
--mode)
COMPREPLY=($(compgen -W "--help init config final" -- $cur_word))
;;
--frequency)
COMPREPLY=($(compgen -W "--help instance always once" -- $cur_word))
;;
schema)
COMPREPLY=($(compgen -W "--help --config-file --doc --annotate" -- $cur_word))
;;
show)
COMPREPLY=($(compgen -W "--help --format --infile --outfile" -- $cur_word))
;;
esac
;;
*)
COMPREPLY=()
;;
esac
}
complete -F _cloudinit_complete cloud-init
# vi: syntax=bash expandtab
# Copyright (C) 2017 Canonical Ltd.
#
# This file is part of cloud-init. See LICENSE file for license information.
import argparse
import re
import sys
from cloudinit.util import json_dumps
from . import dump
from . import show
def get_parser(parser=None):
if not parser:
parser = argparse.ArgumentParser(
prog='cloudinit-analyze',
description='Devel tool: Analyze cloud-init logs and data')
subparsers = parser.add_subparsers(title='Subcommands', dest='subcommand')
subparsers.required = True
parser_blame = subparsers.add_parser(
'blame', help='Print list of executed stages ordered by time to init')
parser_blame.add_argument(
'-i', '--infile', action='store', dest='infile',
default='/var/log/cloud-init.log',
help='specify where to read input.')
parser_blame.add_argument(
'-o', '--outfile', action='store', dest='outfile', default='-',
help='specify where to write output. ')
parser_blame.set_defaults(action=('blame', analyze_blame))
parser_show = subparsers.add_parser(
'show', help='Print list of in-order events during execution')
parser_show.add_argument('-f', '--format', action='store',
dest='print_format', default='%I%D @%Es +%ds',
help='specify formatting of output.')
parser_show.add_argument('-i', '--infile', action='store',
dest='infile', default='/var/log/cloud-init.log',
help='specify where to read input.')
parser_show.add_argument('-o', '--outfile', action='store',
dest='outfile', default='-',
help='specify where to write output.')
parser_show.set_defaults(action=('show', analyze_show))
parser_dump = subparsers.add_parser(
'dump', help='Dump cloud-init events in JSON format')
parser_dump.add_argument('-i', '--infile', action='store',
dest='infile', default='/var/log/cloud-init.log',
help='specify where to read input. ')
parser_dump.add_argument('-o', '--outfile', action='store',
dest='outfile', default='-',
help='specify where to write output. ')
parser_dump.set_defaults(action=('dump', analyze_dump))
return parser
def analyze_blame(name, args):
"""Report a list of records sorted by largest time delta.
For example:
30.210s (init-local) searching for datasource
8.706s (init-network) reading and applying user-data
166ms (modules-config) ....
807us (modules-final) ...
We generate event records parsing cloud-init logs, formatting the output
and sorting by record data ('delta')
"""
(infh, outfh) = configure_io(args)
blame_format = ' %ds (%n)'
r = re.compile(r'(^\s+\d+\.\d+)', re.MULTILINE)
for idx, record in enumerate(show.show_events(_get_events(infh),
blame_format)):
srecs = sorted(filter(r.match, record), reverse=True)
outfh.write('-- Boot Record %02d --\n' % (idx + 1))
outfh.write('\n'.join(srecs) + '\n')
outfh.write('\n')
outfh.write('%d boot records analyzed\n' % (idx + 1))
def analyze_show(name, args):
"""Generate output records using the 'standard' format to printing events.
Example output follows:
Starting stage: (init-local)
...
Finished stage: (init-local) 0.105195 seconds
Starting stage: (init-network)
...
Finished stage: (init-network) 0.339024 seconds
Starting stage: (modules-config)
...
Finished stage: (modules-config) 0.NNN seconds
Starting stage: (modules-final)
...
Finished stage: (modules-final) 0.NNN seconds
"""
(infh, outfh) = configure_io(args)
for idx, record in enumerate(show.show_events(_get_events(infh),
args.print_format)):
outfh.write('-- Boot Record %02d --\n' % (idx + 1))
outfh.write('The total time elapsed since completing an event is'
' printed after the "@" character.\n')
outfh.write('The time the event takes is printed after the "+" '
'character.\n\n')
outfh.write('\n'.join(record) + '\n')
outfh.write('%d boot records analyzed\n' % (idx + 1))
def analyze_dump(name, args):
"""Dump cloud-init events in json format"""
(infh, outfh) = configure_io(args)
outfh.write(json_dumps(_get_events(infh)) + '\n')
def _get_events(infile):
rawdata = None
events, rawdata = show.load_events(infile, None)
if not events:
events, _ = dump.dump_events(rawdata=rawdata)
return events
def configure_io(args):
"""Common parsing and setup of input/output files"""
if args.infile == '-':
infh = sys.stdin
else:
try:
infh = open(args.infile, 'r')
except OSError:
sys.stderr.write('Cannot open file %s\n' % args.infile)
sys.exit(1)
if args.outfile == '-':
outfh = sys.stdout
else:
try:
outfh = open(args.outfile, 'w')
except OSError:
sys.stderr.write('Cannot open file %s\n' % args.outfile)
sys.exit(1)
return (infh, outfh)
if __name__ == '__main__':
parser = get_parser()
args = parser.parse_args()
(name, action_functor) = args.action
action_functor(name, args)
# vi: ts=4 expandtab
# This file is part of cloud-init. See LICENSE file for license information.
import calendar
from datetime import datetime
import sys
from cloudinit import util
stage_to_description = {
'finished': 'finished running cloud-init',
'init-local': 'starting search for local datasources',
'init-network': 'searching for network datasources',
'init': 'searching for network datasources',
'modules-config': 'running config modules',
'modules-final': 'finalizing modules',
'modules': 'running modules for',
'single': 'running single module ',
}
# logger's asctime format
CLOUD_INIT_ASCTIME_FMT = "%Y-%m-%d %H:%M:%S,%f"
# journctl -o short-precise