Commit 9a03b72a authored by Guillaume Binet's avatar Guillaume Binet Committed by Guillaume Binet (argo.ai)

Yapsictomy - part 1: botplugins and flows (#1219)

* Intermediate state.

* Added a separate plugin info dataclass

* add dataclasses as dep from python < 3.7

* version2array -> version2tuple

* version2tuple -> version2array

* Pass on PluginInfo

* temp

* another pass, running not working

* State where it starts to load plugins

* Cleanup for flows.

* A little bit more tests pass.

* Every tests passes except flows.

* Working on flows

* All tests passes.

* make the linter happy.

* Dataclasses backport doesn't support 3.4+3.5.

* remove 3.4 and 3.5

* error in the travis.yml.
parent 60d8cde7
......@@ -2,10 +2,6 @@ sudo: false
language: python
matrix:
include:
- python: 3.4
env: TOXENV=py34
- python: 3.5
env: TOXENV=py35
- python: 3.6
env: TOXENV=py36
- python: 3.6
......
......@@ -28,13 +28,12 @@ class Backup(BotPlugin):
f.write('log.info("Restoring plugins data.")\n')
f.write('bot.plugin_manager.update_dynamic_plugins()\n')
for plug in self._bot.plugin_manager.getAllPlugins():
pobj = plug.plugin_object
if pobj._store:
f.write('pobj = bot.plugin_manager.get_plugin_by_name("' + plug.name + '").plugin_object\n')
for plugin in self._bot.plugin_manager.plugins.values():
if plugin._store:
f.write('pobj = bot.plugin_manager.plugins["' + plugin.name + '"]\n')
f.write('pobj.init_storage()\n')
for key, value in pobj.items():
for key, value in plugin:
f.write('pobj["' + key + '"] = ' + repr(value) + '\n')
f.write('pobj.close_storage()\n')
......
......@@ -46,7 +46,7 @@ class Health(BotPlugin):
pm = self._bot.plugin_manager
all_blacklisted = pm.get_blacklisted_plugin()
all_loaded = pm.get_all_active_plugin_names()
all_attempted = sorted([p.name for p in pm.all_candidates])
all_attempted = sorted(pm.plugin_infos.keys())
plugins_statuses = []
for name in all_attempted:
if name in all_blacklisted:
......
......@@ -31,7 +31,7 @@ class Plugins(BotPlugin):
yield 'Some plugins are generating errors:\n' + '\n'.join(errors.values())
# if the load of the plugin failed, uninstall cleanly teh repo
for path in errors.keys():
if path.startswith(local_path):
if str(path).startswith(local_path):
yield 'Removing %s as it did not load correctly.' % local_path
shutil.rmtree(local_path)
else:
......
......@@ -5,12 +5,12 @@ import sys
import requests
from errbot import BotPlugin
from errbot.utils import version2array
from errbot.utils import version2tuple
from errbot.version import VERSION
HOME = 'http://version.errbot.io/'
installed_version = version2array(VERSION)
installed_version = version2tuple(VERSION)
PY_VERSION = '.'.join(str(e) for e in sys.version_info[:3])
......@@ -38,7 +38,7 @@ class VersionChecker(BotPlugin):
try:
current_version_txt = requests.get(HOME, params={'errbot': VERSION, 'python': PY_VERSION}).text.strip()
self.log.debug("Tested current Errbot version and it is " + current_version_txt)
current_version = version2array(current_version_txt)
current_version = version2tuple(current_version_txt)
if installed_version < current_version:
self.log.debug('A new version %s has been found, notify the admins !' % current_version)
self.warn_admins(
......
from configparser import ConfigParser
from dataclasses import dataclass
from errbot.utils import version2tuple
from pathlib import Path
from typing import Tuple, List
from configparser import Error as ConfigParserError
VersionType = Tuple[int, int, int]
@dataclass
class PluginInfo:
name: str
module: str
doc: str
core: bool
python_version: VersionType
errbot_minversion: VersionType
errbot_maxversion: VersionType
dependencies: List[str]
location: Path = None
@staticmethod
def load(plugfile_path: Path) -> 'PluginInfo':
with plugfile_path.open(encoding='utf-8') as plugfile:
return PluginInfo.load_file(plugfile, plugfile_path)
@staticmethod
def load_file(plugfile, location: Path) -> 'PluginInfo':
cp = ConfigParser()
cp.read_file(plugfile)
pi = PluginInfo.parse(cp)
pi.location = location
return pi
@staticmethod
def parse(config: ConfigParser) -> 'PluginInfo':
"""
Throws ConfigParserError with a meaningful message if the ConfigParser doesn't contain the minimal
information required.
"""
name = config.get('Core', 'Name')
module = config.get('Core', 'Module')
core = config.get('Core', 'Core', fallback='false').lower() == 'true'
doc = config.get('Documentation', 'Description', fallback=None)
python_version = config.get('Python', 'Version', fallback=None)
# Old format backward compatibility
if python_version:
if python_version in ('2+', '3'):
python_version = (3, 0, 0)
elif python_version == '2':
python_version = (2, 0, 0)
else:
try:
python_version = tuple(version2tuple(python_version)[0:3]) # We can ignore the alpha/beta part.
except ValueError as ve:
raise ConfigParserError('Invalid Python Version format: %s (%s)' % (python_version, ve))
min_version = config.get("Errbot", "Min", fallback=None)
max_version = config.get("Errbot", "Max", fallback=None)
try:
if min_version:
min_version = version2tuple(min_version)
except ValueError as ve:
raise ConfigParserError('Invalid Errbot min version format: %s (%s)' % (min_version, ve))
try:
if max_version:
max_version = version2tuple(max_version)
except ValueError as ve:
raise ConfigParserError('Invalid Errbot max version format: %s (%s)' % (max_version, ve))
depends_on = config.get('Core', 'DependsOn', fallback=None)
deps = [name.strip() for name in depends_on.split(',')] if depends_on else []
return PluginInfo(name, module, doc, core, python_version, min_version, max_version, deps)
This diff is collapsed.
import logging
import os
from errbot.plugin_info import PluginInfo
from jinja2 import Environment, FileSystemLoader
from pathlib import Path
log = logging.getLogger(__name__)
def make_templates_path(root):
return os.path.join(root, 'templates')
def make_templates_path(root: Path) -> Path:
return root / 'templates'
system_templates_path = make_templates_path(os.path.dirname(__file__))
system_templates_path = str(make_templates_path(Path(__file__).parent))
template_path = [system_templates_path]
env = Environment(loader=FileSystemLoader(template_path),
trim_blocks=True,
......@@ -21,25 +23,22 @@ def tenv():
return env
def make_templates_from_plugin_path(plugin_path):
return make_templates_path(os.path.dirname(plugin_path))
def add_plugin_templates_path(path):
def add_plugin_templates_path(plugin_info: PluginInfo):
global env
tmpl_path = make_templates_from_plugin_path(path)
if os.path.exists(tmpl_path):
tmpl_path = make_templates_path(plugin_info.location.parent)
if tmpl_path.exists():
log.debug("Templates directory found for this plugin [%s]" % tmpl_path)
template_path.append(tmpl_path)
template_path.append(str(tmpl_path)) # for webhooks
# Ditch and recreate a new templating environment
env = Environment(loader=FileSystemLoader(template_path), autoescape=True)
return
log.debug("No templates directory found for this plugin [Looking for %s]" % tmpl_path)
def remove_plugin_templates_path(path):
def remove_plugin_templates_path(plugin_info: PluginInfo):
global env
tmpl_path = make_templates_from_plugin_path(path)
tmpl_path = str(make_templates_path(plugin_info.location.parent))
if tmpl_path in template_path:
template_path.remove(tmpl_path)
# Ditch and recreate a new templating environment
......
......@@ -77,7 +77,7 @@ def get_class_for_method(meth):
INVALID_VERSION_EXCEPTION = 'version %s in not in format "x.y.z" or "x.y.z-{beta,alpha,rc1,rc2...}" for example "1.2.2"'
def version2array(version):
def version2tuple(version):
vsplit = version.split('-')
if len(vsplit) == 2:
......@@ -103,7 +103,7 @@ def version2array(version):
if len(response) != 4:
raise ValueError(INVALID_VERSION_EXCEPTION % version)
return response
return tuple(response)
def unescape_xml(text):
......
......@@ -22,6 +22,7 @@ from setuptools import setup, find_packages
py_version = sys.version_info[:2]
PY35_OR_GREATER = py_version >= (3, 5)
PY37_OR_GREATER = py_version >= (3, 7)
ON_WINDOWS = system() == 'Windows'
......@@ -48,6 +49,8 @@ deps = ['webtest',
if not PY35_OR_GREATER:
deps += ['typing', ] # backward compatibility for 3.3 and 3.4
if not PY37_OR_GREATER:
deps += ['dataclasses'] # backward compatibility for 3.3->3.6 for dataclasses
if not ON_WINDOWS:
deps += ['daemonize']
......
# coding=utf-8
import sys
import logging
from pathlib import Path
from tempfile import mkdtemp
from os.path import sep
......@@ -276,7 +277,7 @@ def dummy_execute_and_send():
example_message.to = dummy.build_identifier('err')
assets_path = os.path.join(os.path.dirname(__file__), 'assets')
templating.template_path.append(templating.make_templates_path(assets_path))
templating.template_path.append(str(templating.make_templates_path(Path(assets_path))))
templating.env = templating.Environment(loader=templating.FileSystemLoader(templating.template_path))
return dummy, example_message
......
......@@ -153,8 +153,7 @@ def test_broken_plugin(testbot):
tgz = os.path.join(tempd, "borken.tar.gz")
with tarfile.open(tgz, "w:gz") as tar:
tar.add(borken_plugin_dir, arcname='borken')
assert 'Installing' in testbot.exec_command('!repos install file://' + tgz,
timeout=120)
assert 'Installing' in testbot.exec_command('!repos install file://' + tgz, timeout=120)
assert 'import borken # fails' in testbot.pop_message()
assert 'as it did not load correctly.' in testbot.pop_message()
assert 'Plugins reloaded.' in testbot.pop_message()
......
......@@ -13,7 +13,7 @@ def test_if_all_loaded_by_default(testbot):
def test_single_dependency(testbot):
pm = testbot.bot.plugin_manager
for p in ('Single', 'Parent1', 'Parent2'):
pm.deactivate_plugin_by_name(p)
pm.deactivate_plugin(p)
# everything should be gone
plug_names = pm.get_all_active_plugin_names()
......@@ -34,7 +34,7 @@ def test_double_dependency(testbot):
pm = testbot.bot.plugin_manager
all = ('Double', 'Parent1', 'Parent2')
for p in all:
pm.deactivate_plugin_by_name(p)
pm.deactivate_plugin(p)
pm.activate_plugin('Double')
plug_names = pm.get_all_active_plugin_names()
......@@ -46,12 +46,12 @@ def test_dependency_retrieval(testbot):
assert 'youpi' in testbot.exec_command('!depfunc')
def test_direct_cicular_dependency(testbot):
def test_direct_circular_dependency(testbot):
plug_names = testbot.bot.plugin_manager.get_all_active_plugin_names()
assert 'Circular1' not in plug_names
def test_indirect_cicular_dependency(testbot):
def test_indirect_circular_dependency(testbot):
plug_names = testbot.bot.plugin_manager.get_all_active_plugin_names()
assert 'Circular2' not in plug_names
assert 'Circular3' not in plug_names
......
import sys
import pytest
from io import StringIO
from pathlib import Path
from errbot.plugin_info import PluginInfo
plugfile_base = Path(__file__).absolute().parent / 'config_plugin'
plugfile_path = plugfile_base / 'config.plug'
def test_load_from_plugfile_path():
pi = PluginInfo.load(plugfile_path)
assert pi.name == 'Config'
assert pi.module == 'config'
assert pi.doc is None
assert pi.python_version == (3, 0, 0)
assert pi.errbot_minversion is None
assert pi.errbot_maxversion is None
@pytest.mark.parametrize('test_input,expected', [
('2', (2, 0, 0)),
('2+', (3, 0, 0)),
('3', (3, 0, 0)),
('1.2.3', (1, 2, 3)),
('1.2.3-beta', (1, 2, 3)),
])
def test_python_version_parse(test_input, expected):
f = StringIO("""
[Core]
Name = Config
Module = config
[Python]
Version = %s
""" % test_input)
assert PluginInfo.load_file(f, None).python_version == expected
def test_doc():
f = StringIO("""
[Core]
Name = Config
Module = config
[Documentation]
Description = something
""")
assert PluginInfo.load_file(f, None).doc == 'something'
def test_errbot_version():
f = StringIO("""
[Core]
Name = Config
Module = config
[Errbot]
Min = 1.2.3
Max = 4.5.6-beta
""")
info = PluginInfo.load_file(f, None)
assert info.errbot_minversion == (1, 2, 3, sys.maxsize)
assert info.errbot_maxversion == (4, 5, 6, 0)
import os
import pytest
import tempfile
from configparser import ConfigParser
from errbot import plugin_manager
from errbot.plugin_info import PluginInfo
from errbot.plugin_manager import IncompatiblePluginException
from errbot.utils import find_roots, collect_roots
CORE_PLUGINS = plugin_manager.CORE_PLUGINS
......@@ -80,44 +83,43 @@ def test_ignore_dotted_directories():
assert collect_roots((CORE_PLUGINS, root)) == {CORE_PLUGINS, }
def dummy_config_parser() -> ConfigParser:
cp = ConfigParser()
cp.add_section('Core')
cp.set('Core', 'Name', 'dummy')
cp.set('Core', 'Module', 'dummy')
cp.add_section('Errbot')
return cp
def test_errbot_version_check():
real_version = plugin_manager.VERSION
too_high_min_1 = ConfigParser()
too_high_min_1.add_section('Errbot')
too_high_min_1 = dummy_config_parser()
too_high_min_1.set('Errbot', 'Min', '1.6.0')
too_high_min_2 = ConfigParser()
too_high_min_2.add_section('Errbot')
too_high_min_2 = dummy_config_parser()
too_high_min_2.set('Errbot', 'Min', '1.6.0')
too_high_min_2.set('Errbot', 'Max', '2.0.0')
too_low_max_1 = ConfigParser()
too_low_max_1.add_section('Errbot')
too_low_max_1 = dummy_config_parser()
too_low_max_1.set('Errbot', 'Max', '1.0.1-beta')
too_low_max_2 = ConfigParser()
too_low_max_2.add_section('Errbot')
too_low_max_2 = dummy_config_parser()
too_low_max_2.set('Errbot', 'Min', '0.9.0-rc2')
too_low_max_2.set('Errbot', 'Max', '1.0.1-beta')
ok1 = ConfigParser() # no section
ok2 = ConfigParser()
ok2.add_section('Errbot') # empty section
ok1 = dummy_config_parser() # empty section
ok3 = ConfigParser()
ok3.add_section('Errbot')
ok3.set('Errbot', 'Min', '1.4.0')
ok2 = dummy_config_parser()
ok2.set('Errbot', 'Min', '1.4.0')
ok4 = ConfigParser()
ok4.add_section('Errbot')
ok4.set('Errbot', 'Max', '1.5.2')
ok3 = dummy_config_parser()
ok3.set('Errbot', 'Max', '1.5.2')
ok5 = ConfigParser()
ok5.add_section('Errbot')
ok5 .set('Errbot', 'Min', '1.2.1')
ok5 .set('Errbot', 'Max', '1.6.1-rc1')
ok4 = dummy_config_parser()
ok4.set('Errbot', 'Min', '1.2.1')
ok4.set('Errbot', 'Max', '1.6.1-rc1')
try:
plugin_manager.VERSION = '1.5.2'
......@@ -125,9 +127,12 @@ def test_errbot_version_check():
too_high_min_2,
too_low_max_1,
too_low_max_2):
assert not plugin_manager.check_errbot_plug_section('should_fail', config)
pi = PluginInfo.parse(config)
with pytest.raises(IncompatiblePluginException):
plugin_manager.check_errbot_version(pi)
for config in (ok1, ok2, ok3, ok4, ok5):
assert plugin_manager.check_errbot_plug_section('should_work', config)
for config in (ok1, ok2, ok3, ok4):
pi = PluginInfo.parse(config)
plugin_manager.check_errbot_version(pi)
finally:
plugin_manager.VERSION = real_version
......@@ -24,7 +24,7 @@ log = logging.getLogger(__name__)
('2.0.0-beta', '2.0.1'),
])
def test_version_check(v1, v2):
assert version2array(v1) < version2array(v2)
assert version2tuple(v1) < version2tuple(v2)
@pytest.mark.parametrize('version', [
......@@ -36,7 +36,7 @@ def test_version_check(v1, v2):
])
def test_version_check_negative(version):
with pytest.raises(ValueError):
version2array(version)
version2tuple(version)
def test_formattimedelta():
......
[tox]
envlist = py34,py35,py36,codestyle,pypi-lint
envlist = py36,codestyle,pypi-lint
skip_missing_interpreters = True
[testenv]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment