Commit 2081529b authored by Guillaume Binet's avatar Guillaume Binet

Refactored Errbot to inject the storage plugin.

parent 2ba9d45b
......@@ -262,7 +262,7 @@ class TestBackend(ErrBot):
log.debug("Trigger disconnect callback")
self.disconnect_callback()
log.debug("Trigger shutdown")
self.shutdown()
self.plugin_manager.shutdown()
def connect(self):
return
......@@ -342,6 +342,7 @@ class TestBot(object):
__import__('errbot.config-template')
config = sys.modules['errbot.config-template']
tempdir = mkdtemp()
config.STORAGE = 'Memory'
config.BOT_DATA_DIR = tempdir
config.BOT_LOG_FILE = tempdir + sep + 'log.txt'
......
......@@ -33,7 +33,7 @@ class BotPluginBase(StoreMixin):
""" This should be eventually moved back to __init__ once plugin will forward correctly their params.
"""
self._bot = bot
self.plugin_dir = bot.plugin_dir
self.plugin_dir = bot.plugin_manager.plugin_dir
@property
def mode(self) -> str:
......@@ -69,9 +69,7 @@ class BotPluginBase(StoreMixin):
def init_storage(self) -> None:
classname = self.__class__.__name__
log.debug('Init storage for %s' % classname)
filename = os.path.join(self.bot_config.BOT_DATA_DIR, PLUGINS_SUBDIR, classname + '.db')
log.debug('Loading %s' % filename)
self.open_storage(filename)
self.open_storage(self._bot.storage_plugin, classname)
def activate(self) -> None:
"""
......
......@@ -15,13 +15,13 @@ class Backup(BotPlugin):
with open(filename, 'w') as f:
f.write('## This file is not executable on its own. use err.py -r FILE to restore your bot.\n\n')
f.write('log.info("Restoring core configs.")\n')
for key in self._bot: # don't mimic that in real plugins, this is core only.
f.write('bot["'+key+'"] = ' + repr(self._bot[key]) + '\n')
for key in self._bot.plugin_manager: # don't mimic that in real plugins, this is core only.
f.write('bot.plugin_manager["'+key+'"] = ' + repr(self._bot[key]) + '\n')
f.write('log.info("Installing plugins.")\n')
f.write('if "repos" in bot:\n')
f.write(' for repo in bot["repos"]:\n')
f.write(' errors = bot.install_repo(repo)\n')
f.write('if "repos" in bot.plugin_manager:\n')
f.write(' for repo in bot.plugin_manager["repos"]:\n')
f.write(' errors = bot.plugin_manager.install_repo(repo)\n')
f.write(' for error in errors:\n')
f.write(' log.error(error)\n')
......@@ -30,7 +30,7 @@ class Backup(BotPlugin):
for plug in self._bot.getAllPlugins():
pobj = plug.plugin_object
if pobj.shelf:
f.write('pobj = bot.get_plugin_by_name("' + plug.name + '").plugin_object\n')
f.write('pobj = bot.plugin_manager.get_plugin_by_name("' + plug.name + '").plugin_object\n')
f.write('pobj.init_storage()\n')
for key in pobj.shelf:
......
......@@ -9,7 +9,6 @@ from errbot.utils import format_timedelta
class Health(BotPlugin):
@botcmd(template='status')
def status(self, mess, args):
""" If I am alive I should be able to respond to this one
......@@ -44,9 +43,9 @@ class Health(BotPlugin):
def status_plugins(self, mess, args):
""" shows the plugin status
"""
all_blacklisted = self._bot.get_blacklisted_plugin()
all_loaded = self._bot.get_all_active_plugin_names()
all_attempted = sorted([p.name for p in self._bot.all_candidates])
all_blacklisted = self._bot.plugin_manager.get_blacklisted_plugin()
all_loaded = self._bot.plugin_manager.get_all_active_plugin_names()
all_attempted = sorted([p.name for p in self._bot.plugin_manager.all_candidates])
plugins_statuses = []
for name in all_attempted:
if name in all_blacklisted:
......@@ -56,8 +55,10 @@ class Health(BotPlugin):
plugins_statuses.append(('BD', name))
elif name in all_loaded:
plugins_statuses.append(('A', name))
elif self._bot.get_plugin_obj_by_name(name) is not None and self._bot.get_plugin_obj_by_name(
name).get_configuration_template() is not None and self._bot.get_plugin_configuration(name) is None:
elif self._bot.plugin_manager.get_plugin_obj_by_name(
name) is not None and self._bot.plugin_manager.get_plugin_obj_by_name(
name).get_configuration_template() is not None and self._bot.plugin_manager.get_plugin_configuration(
name) is None:
plugins_statuses.append(('C', name))
else:
plugins_statuses.append(('D', name))
......@@ -76,7 +77,7 @@ class Health(BotPlugin):
def restart(self, mess, args):
""" Restart the bot. """
self.send(mess.frm, "Deactivating all the plugins...")
self._bot.deactivate_all_plugins()
self._bot.plugin_manager.deactivate_all_plugins()
self.send(mess.frm, "Restarting")
self._bot.shutdown()
global_restart()
......@@ -92,7 +93,7 @@ class Health(BotPlugin):
return "Use `!killbot really` if you really want to shutdown the bot."
self.send(mess.frm, "Dave, I can see you are really upset about this...")
self._bot.deactivate_all_plugins()
self._bot.plugin_manager.deactivate_all_plugins()
self.send(mess.frm, "I know I have made some very poor decisions recently...")
self.send(mess.frm, "Daisy, Daaaaiseey...")
self._bot.shutdown()
......
......@@ -23,7 +23,7 @@ class Plugins(BotPlugin):
"""
if not args.strip():
return "You should have an urls/git repo argument"
errors = self._bot.install_repo(args)
errors = self._bot.plugin_manager.install_repo(args)
if errors:
self.send(mess.frm, 'Some plugins are generating errors:\n' + '\n'.join(errors),
message_type=mess.type)
......@@ -34,7 +34,7 @@ class Plugins(BotPlugin):
"%s. Refreshing the plugins commands..." % args),
message_type=mess.type
)
loading_errors = self._bot.activate_non_started_plugins()
loading_errors = self._bot.plugin_manager.activate_non_started_plugins()
if loading_errors:
return loading_errors
return "Plugins reloaded without any error."
......@@ -47,21 +47,21 @@ class Plugins(BotPlugin):
yield "You should have a repo name as argument"
return
repos = self._bot.get_installed_plugin_repos()
repos = self._bot.plugin_manager.get_installed_plugin_repos()
if args not in repos:
yield "This repo is not installed check with " + self._bot.prefix + "repos the list of installed ones"
return
plugin_path = path.join(self._bot.plugin_dir, args)
for plugin in self._bot.getAllPlugins():
plugin_path = path.join(self._bot.plugin_manager.plugin_dir, args)
for plugin in self._bot.plugin_manager.getAllPlugins():
if plugin.path.startswith(plugin_path):
yield 'Removing %s...' % plugin.name
self._bot.remove_plugin(plugin)
self._bot.plugin_manager.remove_plugin(plugin)
shutil.rmtree(plugin_path)
repos.pop(args)
self._bot.set_plugin_repos(repos)
self._bot.plugin_manager.set_plugin_repos(repos)
yield 'Repo %s removed.' % args
......@@ -184,9 +184,9 @@ class Plugins(BotPlugin):
{'LOGIN': 'my@email.com', 'PASSWORD': 'myrealpassword', 'DIRECTORY': '/tmp'}
"""
plugin_name = args[0]
if self._bot.is_plugin_blacklisted(plugin_name):
if self._bot.plugin_manager.is_plugin_blacklisted(plugin_name):
return 'Load this plugin first with ' + self._bot.prefix + 'load %s' % plugin_name
obj = self._bot.get_plugin_obj_by_name(plugin_name)
obj = self._bot.plugin_manager.get_plugin_obj_by_name(plugin_name)
if obj is None:
return 'Unknown plugin or the plugin could not load %s' % plugin_name
template_obj = obj.get_configuration_template()
......@@ -199,7 +199,7 @@ class Plugins(BotPlugin):
"```\n{prefix}plugin config {plugin_name} \n{config}\n```").format(
prefix=self._bot.prefix, plugin_name=plugin_name, config=pformat(template_obj))
current_config = self._bot.get_plugin_configuration(plugin_name)
current_config = self._bot.plugin_manager.get_plugin_configuration(plugin_name)
if current_config:
response += ("\n\nCurrent configuration:\n\n"
"```\n{prefix}plugin config {plugin_name} \n{config}\n```").format(
......@@ -215,13 +215,13 @@ class Plugins(BotPlugin):
if type(real_config_obj) != type(template_obj):
return 'It looks fishy, your config type is not the same as the template !'
self._bot.set_plugin_configuration(plugin_name, real_config_obj)
self._bot.deactivate_plugin(plugin_name)
self._bot.plugin_manager.set_plugin_configuration(plugin_name, real_config_obj)
self._bot.plugin_manager.deactivate_plugin(plugin_name)
try:
self._bot.activate_plugin(plugin_name)
self._bot.plugin_manager.activate_plugin(plugin_name)
except PluginConfigurationException as ce:
self.log.debug('Invalid configuration for the plugin, reverting the plugin to unconfigured')
self._bot.set_plugin_configuration(plugin_name, None)
self._bot.plugin_manager.set_plugin_configuration(plugin_name, None)
return 'Incorrect plugin configuration: %s' % ce
return 'Plugin configuration done.'
......@@ -234,9 +234,9 @@ class Plugins(BotPlugin):
(blacklisted) plugins.
"""
if active_only:
all_plugins = self._bot.get_all_active_plugin_names()
all_plugins = self._bot.plugin_manager.get_all_active_plugin_names()
else:
all_plugins = self._bot.get_all_plugin_names()
all_plugins = self._bot.plugin_manager.get_all_plugin_names()
return "\n".join(("- " + plugin for plugin in all_plugins))
# noinspection PyUnusedLocal
......@@ -248,16 +248,16 @@ class Plugins(BotPlugin):
yield ("Please tell me which of the following plugins to reload:\n"
"{}".format(self.formatted_plugin_list(active_only=False)))
return
if name not in self._bot.get_all_plugin_names():
if name not in self._bot.plugin_manager.get_all_plugin_names():
yield ("{} isn't a valid plugin name. The current plugins are:\n"
"{}".format(name, self.formatted_plugin_list(active_only=False)))
return
if name not in self._bot.get_all_active_plugin_names():
if name not in self._bot.plugin_manager.get_all_active_plugin_names():
yield (("Warning: plugin %s is currently not activated. " +
"Use !plugin activate %s to activate it.") % (name, name))
self._bot.reload_plugin_by_name(name)
self._bot.plugin_manager.reload_plugin_by_name(name)
yield "Plugin %s reloaded." % name
......@@ -269,13 +269,13 @@ class Plugins(BotPlugin):
if not args:
return ("Please tell me which of the following plugins to activate:\n"
"{}".format(self.formatted_plugin_list(active_only=False)))
if args not in self._bot.get_all_plugin_names():
if args not in self._bot.plugin_manager.get_all_plugin_names():
return ("{} isn't a valid plugin name. The current plugins are:\n"
"{}".format(args, self.formatted_plugin_list(active_only=False)))
if args in self._bot.get_all_active_plugin_names():
if args in self._bot.plugin_manager.get_all_active_plugin_names():
return "{} is already activated.".format(args)
return self._bot.activate_plugin(args)
return self._bot.plugin_manager.activate_plugin(args)
# noinspection PyUnusedLocal
@botcmd(admin_only=True)
......@@ -285,37 +285,37 @@ class Plugins(BotPlugin):
if not args:
return ("Please tell me which of the following plugins to deactivate:\n"
"{}".format(self.formatted_plugin_list(active_only=False)))
if args not in self._bot.get_all_plugin_names():
if args not in self._bot.plugin_manager.get_all_plugin_names():
return ("{} isn't a valid plugin name. The current plugins are:\n"
"{}".format(args, self.formatted_plugin_list(active_only=False)))
if args not in self._bot.get_all_active_plugin_names():
if args not in self._bot.plugin_manager.get_all_active_plugin_names():
return "{} is already deactivated.".format(args)
return self._bot.deactivate_plugin(args)
return self._bot.plugin_manager.deactivate_plugin(args)
# noinspection PyUnusedLocal
@botcmd(admin_only=True)
def plugin_blacklist(self, mess, args):
"""Blacklist a plugin so that it will not be loaded automatically during bot startup.
If the plugin is currently activated, it will deactiveate it first."""
if args not in self._bot.get_all_plugin_names():
if args not in self._bot.plugin_manager.get_all_plugin_names():
return ("{} isn't a valid plugin name. The current plugins are:\n"
"{}".format(args, self.formatted_plugin_list(active_only=False)))
if args in self._bot.get_all_active_plugin_names():
self._bot.deactivate_plugin(args)
if args in self._bot.plugin_manager.get_all_active_plugin_names():
self._bot.plugin_manager.deactivate_plugin(args)
return self._bot.blacklist_plugin(args)
return self._bot.plugin_manager.blacklist_plugin(args)
# noinspection PyUnusedLocal
@botcmd(admin_only=True)
def plugin_unblacklist(self, mess, args):
"""Remove a plugin from the blacklist"""
if args not in self._bot.get_all_plugin_names():
if args not in self._bot.plugin_manager.get_all_plugin_names():
return ("{} isn't a valid plugin name. The current plugins are:\n"
"{}".format(args, self.formatted_plugin_list(active_only=False)))
if args not in self._bot.get_all_active_plugin_names():
self._bot.activate_plugin(args)
if args not in self._bot.plugin_manager.get_all_active_plugin_names():
self._bot.plugin_manager.activate_plugin(args)
return self._bot.unblacklist_plugin(args)
return self._bot.plugin_manager.unblacklist_plugin(args)
......@@ -61,7 +61,7 @@ def bot_config_defaults(config):
# noinspection PyAbstractClass
class ErrBot(Backend, BotPluginManager):
class ErrBot(Backend):
""" ErrBot is the layer of Err that takes care of the plugin management and dispatching
"""
__errdoc__ = """ Commands related to the bot administration """
......@@ -72,7 +72,6 @@ class ErrBot(Backend, BotPluginManager):
def __init__(self, bot_config):
log.debug("ErrBot init.")
super().__init__(bot_config)
self._init_plugin_manager(bot_config)
self.bot_config = bot_config
self.prefix = bot_config.BOT_PREFIX
if bot_config.BOT_ASYNC:
......@@ -87,6 +86,16 @@ class ErrBot(Backend, BotPluginManager):
self.bot_alt_prefixes = tuple(prefix.lower() for prefix in bot_config.BOT_ALT_PREFIXES)
else:
self.bot_alt_prefixes = bot_config.BOT_ALT_PREFIXES
self.plugin_manager = None
self.storage_plugin = None
def attach_plugin_manager(self, plugin_manager):
self.plugin_manager = plugin_manager
plugin_manager.attach_bot(self)
def attach_storage_plugin(self, storage_plugin):
# the storage_plugin is needed by the plugins
self.storage_plugin = storage_plugin
@property
def all_commands(self):
......@@ -105,7 +114,7 @@ class ErrBot(Backend, BotPluginManager):
:param *args: Passed to the callback function.
:param **kwargs: Passed to the callback function.
"""
for plugin in self.get_all_active_plugin_objects():
for plugin in self.plugin_manager.get_all_active_plugin_objects():
plugin_name = plugin.__class__.__name__
log.debug("Triggering {} on {}".format(method, plugin_name))
# noinspection PyBroadException
......@@ -177,7 +186,7 @@ class ErrBot(Backend, BotPluginManager):
:param mess: the message to send.
:return: None
"""
for bot in self.get_all_active_plugin_objects():
for bot in self.plugin_manager.get_all_active_plugin_objects():
# noinspection PyBroadException
try:
bot.callback_botmessage(mess)
......@@ -557,10 +566,10 @@ class ErrBot(Backend, BotPluginManager):
def callback_stream(self, stream):
log.info("Initiated an incoming transfer %s" % stream)
Tee(stream, self.get_all_active_plugin_objects()).start()
Tee(stream, self.plugin_manager.get_all_active_plugin_objects()).start()
def signal_connect_to_all_plugins(self):
for bot in self.get_all_active_plugin_objects():
for bot in self.plugin_manager.get_all_active_plugin_objects():
if hasattr(bot, 'callback_connect'):
# noinspection PyBroadException
try:
......@@ -571,7 +580,7 @@ class ErrBot(Backend, BotPluginManager):
def connect_callback(self):
log.info('Activate internal commands')
loading_errors = self.activate_non_started_plugins()
loading_errors = self.plugin_manager.activate_non_started_plugins()
log.info(loading_errors)
log.info('Notifying connection to all the plugins...')
self.signal_connect_to_all_plugins()
......@@ -579,7 +588,7 @@ class ErrBot(Backend, BotPluginManager):
def disconnect_callback(self):
log.info('Disconnect callback, deactivating all the plugins.')
self.deactivate_all_plugins()
self.plugin_manager.deactivate_all_plugins()
def get_doc(self, command):
"""Get command documentation
......
......@@ -2,19 +2,24 @@ from os import path, makedirs
import logging
from errbot.errBot import ErrBot
from errbot.plugin_manager import BotPluginManager
from errbot.specific_plugin_manager import SpecificPluginManager
import sys
from errbot.storage.base import StoragePluginBase
from errbot.utils import PLUGINS_SUBDIR
log = logging.getLogger(__name__)
CORE_BACKENDS = path.join(path.dirname(path.abspath(__file__)), 'backends')
HERE = path.dirname(path.abspath(__file__))
CORE_BACKENDS = path.join(HERE, 'backends')
CORE_STORAGE = path.join(HERE, 'storage')
def setup_bot(backend_name, logger, config, restore=None):
# from here the environment is supposed to be set (daemon / non daemon,
# config.py in the python path )
from .utils import PLUGINS_SUBDIR
from .errBot import bot_config_defaults
bot_config_defaults(config)
......@@ -41,24 +46,35 @@ def setup_bot(backend_name, logger, config, restore=None):
logger.setLevel(config.BOT_LOG_LEVEL)
# make the plugins subdir to store the plugin shelves
d = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR)
if not path.exists(d):
makedirs(d, mode=0o755)
# init the storage plugin
storage_name = getattr(config, 'STORAGE', 'Shelf')
extra_storage_plugins_dir = getattr(config, 'EXTRA_STORAGE_PLUGINS_DIR', None)
spm = SpecificPluginManager(config, 'storage', StoragePluginBase, CORE_STORAGE, extra_storage_plugins_dir)
storage_pluginfo = spm.get_candidate(storage_name)
log.info("Found Storage plugin: '%s'\nDescription: %s" % (storage_pluginfo.name, storage_pluginfo.description))
storage_plugin = spm.get_plugin_by_name(storage_name)
# init the botplugin manager
botplugins_dir = path.join(config.BOT_DATA_DIR, PLUGINS_SUBDIR)
if not path.exists(botplugins_dir):
makedirs(botplugins_dir, mode=0o755)
botpm = BotPluginManager(storage_plugin,
botplugins_dir,
config.BOT_EXTRA_PLUGIN_DIR,
config.AUTOINSTALL_DEPS)
# instanciate the bot
if hasattr(config, 'BOT_EXTRA_BACKEND_DIR'):
extra = config.BOT_EXTRA_BACKEND_DIR
else:
extra = []
bpm = SpecificPluginManager(config, 'backends', ErrBot, CORE_BACKENDS, extra)
# init the backend manager & the bot
extra = getattr(config, 'BOT_EXTRA_BACKEND_DIR', [])
backendpm = SpecificPluginManager(config, 'backends', ErrBot, CORE_BACKENDS, extra)
plug = bpm.get_candidate(backend_name)
backend_plug = backendpm.get_candidate(backend_name)
log.info("Found Backend plugin: '%s'\n\t\t\t\t\t\tDescription: %s" % (plug.name, plug.description))
log.info("Found Backend plugin: '%s'\n\t\t\t\t\t\tDescription: %s" % (backend_plug.name, backend_plug.description))
try:
bot = bpm.get_plugin_by_name(backend_name)
bot = backendpm.get_plugin_by_name(backend_name)
bot.attach_plugin_manager(botpm)
bot.attach_storage_plugin(storage_plugin)
except Exception:
log.exception("Unable to load or configure the backend.")
exit(-1)
......@@ -76,7 +92,7 @@ def setup_bot(backend_name, logger, config, restore=None):
print('Restore complete. You can restart the bot normally')
sys.exit(0)
errors = bot.update_dynamic_plugins()
errors = bot.plugin_manager.update_dynamic_plugins()
if errors:
log.error('Some plugins failed to load:\n' + '\n'.join(errors))
return bot
......
......@@ -15,7 +15,7 @@ from urllib.request import urlopen
import pip
from .botplugin import BotPlugin
from .utils import (version2array, PY3, PY2, find_roots_with_extra,
PLUGINS_SUBDIR, which, human_name_for_git_url)
which, human_name_for_git_url)
from .templating import remove_plugin_templates_path, add_plugin_templates_path
from .version import VERSION
from yapsy.PluginManager import PluginManager
......@@ -194,20 +194,23 @@ class BotPluginManager(PluginManager, StoreMixin):
CONFIGS = b'configs' if PY2 else 'configs'
BL_PLUGINS = b'bl_plugins' if PY2 else 'bl_plugins'
# gbin: making an __init__ here will be tricky with the MRO
# use _init_plugin_manager directly.
def __init__(self, storage_plugin, plugin_dir, extra, autoinstall_deps):
self.bot = None
self.autoinstall_deps = autoinstall_deps
self.extra = extra
def _init_plugin_manager(self, bot_config):
self.plugin_dir = os.path.join(bot_config.BOT_DATA_DIR, PLUGINS_SUBDIR)
self.open_storage(os.path.join(bot_config.BOT_DATA_DIR, 'core.db'))
self.open_storage(storage_plugin, 'core')
self.plugin_dir = plugin_dir
# be sure we have a configs entry for the plugin configurations
if self.CONFIGS not in self:
self[self.CONFIGS] = {}
self.setCategoriesFilter({"bots": BotPlugin})
locator = PluginFileLocator([PluginFileAnalyzerWithInfoFile("info_ext", 'plug')])
locator.disableRecursiveScan() # We do that ourselves
self.setPluginLocator(locator)
super().__init__(categories_filter={"bots": BotPlugin}, plugin_locator=locator)
def attach_bot(self, bot):
self.bot = bot
def instanciateElement(self, element):
""" Override the loading method to inject bot """
......@@ -220,10 +223,10 @@ class BotPluginManager(PluginManager, StoreMixin):
log.warn(('Warning: %s needs to implement __init__(self, *args, **kwargs) '
'and forward them to super().__init__') % element.__name__)
obj = element()
obj._load_bot(self) # sideload the bot
obj._load_bot(self.bot) # sideload the bot
return obj
return element(self)
return element(self.bot)
def get_plugin_by_name(self, name):
return self.getPluginByName(name, 'bots')
......@@ -452,7 +455,7 @@ class BotPluginManager(PluginManager, StoreMixin):
def update_dynamic_plugins(self):
return self.update_plugin_places(
[self.plugin_dir + os.sep + d for d in self.get(self.REPOS, {}).keys()],
self.bot_config.BOT_EXTRA_PLUGIN_DIR, self.bot_config.AUTOINSTALL_DEPS)
self.extra, self.autoinstall_deps)
def activate_non_started_plugins(self):
log.info('Activating all the plugins...')
......
......@@ -41,7 +41,7 @@ class SpecificPluginManager(PluginManager):
self.setCategoriesFilter({category: base_class})
all_plugins_paths = find_roots_with_extra(base_search_dir, extra_search_dirs)
log.info('%s search paths %s', (category, all_plugins_paths))
log.info('%s search paths %s', category, all_plugins_paths)
self.setPluginPlaces(all_plugins_paths)
for entry in all_plugins_paths:
if entry not in sys.path:
......
......@@ -24,52 +24,49 @@ class StoreMixin(MutableMapping):
This class handle the basic needs of bot plugins and core like loading, unloading and creating a storage
"""
def __init__(self, storage=None, storage_config=None):
log.info('Init shelf of %s' % self.__class__.__name__)
self.shelf = None
# if storage is None:
# raise Exception("Storage not specified")
def open_storage(self, path):
if hasattr(self, 'shelf') and self.shelf is not None:
def __init__(self):
log.info('Init storage of %s' % self.__class__.__name__)
self.store = None
def open_storage(self, storage_plugin, namespace):
if hasattr(self, 'store') and self.store is not None:
raise StoreAlreadyOpenError("Storage appears to be opened already")
log.debug("Opening storage file %s" % path)
self.shelf = shelve.DbfilenameShelf(path, protocol=2)
log.info('Opened shelf of %s at %s' % (self.__class__.__name__, path))
log.debug("Opening storage %s" % namespace)
self.store = storage_plugin.open(namespace)
def close_storage(self):
if not hasattr(self, 'shelf') or self.shelf is None:
if not hasattr(self, 'store') or self.store is None:
raise StoreNotOpenError("Storage does not appear to have been opened yet")
self.shelf.close()
self.shelf = None
log.debug('Closed shelf of %s' % self.__class__.__name__)
self.store.close()
self.store = None
log.debug('Closed store of %s' % self.__class__.__name__)
# those are the minimal things to behave like a dictionary with the UserDict.DictMixin
def __getitem__(self, key):
return self.shelf.__getitem__(key)
return self.store.get(key)
def __setitem__(self, key, item):
answer = self.shelf.__setitem__(key, item)
self.shelf.sync()
return answer
return self.store.set(key, item)
def __delitem__(self, key):
answer = self.shelf.__delitem__(key)
self.shelf.sync()
return answer
return self.store.remove(key)
def keys(self):
keys = self.shelf.keys()
keys = self.store.keys()
if PY2:
keys = [key.decode('utf-8') for key in keys]
return keys
def __len__(self):
return len(self.shelf)
return self.store.len()
def __iter__(self):
for i in self.shelf:
for i in self.store.iter():
yield i
def __contains__(self, x):
return x in self.shelf
try:
self.store.get(x)
return True
except KeyError:
return False
......@@ -28,6 +28,24 @@ class StorageBase(object):
"""
pass
@abstractmethod
def remove(self, key: str) -> None:
"""
Remove key. Raises KeyError if the key doesn't exist.
The caller of get will protect against get on non open.
:param key: the key
"""
pass
@abstractmethod
def len(self) -> int:
"""
:return: the number of keys set.
"""
pass
@abstractmethod
def close(self) -> None:
"""
......@@ -42,7 +60,7 @@ class StoragePluginBase(object):
Base to implement a storage plugin.
This is a factory for the namespaces.
"""
def init(self, bot_config):
def __init__(self, bot_config):
self._storage_config = getattr(bot_config, 'STORAGE_CONFIG', {})
@abstractmethod
......
......@@ -12,13 +12,21 @@ class MemoryStorage(StorageBase):
self.root = ROOTS.get(namespace, {})
def get(self, key: str) -> Any:
if key in self.root:
return self.root[key]
raise KeyError("%s doesn't exist." % key)
if key not in self.root:
raise KeyError("%s doesn't exist." % key)
return self.root[key]
def set(self, key: str, value: Any) -> None:
self.root[key] = value
def remove(self, key: str):
if key not in self.root:
raise KeyError("%s doesn't exist." % key)
del self.root[key]
def len(self):
return len(self.root)
def close(self) -> None:
ROOTS[self.namespace] = self.root
......
import logging
from typing import Any, Mapping
import shelve
import os
from errbot.storage.base import StorageBase, StoragePluginBase
......@@ -17,9 +16,17 @@ class ShelfStorage(StorageBase):
def get(self, key: str) -> Any:
return self.shelf[key]
def remove(self, key: str):
if key not in self.shelf:
raise KeyError("%s doesn't exist." % key)
del self.root[key]
def set(self, key: str, value: Any) -> None:
self.shelf[key] = value
def len(self):
return len(self.shelf)
def close(self) -> None:
self.shelf.close()
self.shelf = None
......@@ -27,13 +34,11 @@ class ShelfStorage(StorageBase):
class ShelfStoragePlugin(StoragePluginBase):
def __init__(self, bot_config):
super().__init__(bot_config)
if 'basedir' not in self._storage_config:
self._storage_config['basedir'] = d
def open(self, namespace: str) -> StorageBase:
config = self._storage_config
if 'basedir' not in self.config:
raise Exception('no basedir specified in the shelfstorage config.')
if 'compatibilitymode' in config and config['compatibilitymode']:
# originally errbot stores plugins per dir.
return ShelfStorage(os.path.join(config['basedir'], namespace, namespace))
return ShelfStorage(os.path.join(config['basedir'], namespace))
return ShelfStorage(os.path.join(config['basedir'], namespace + '.db'))
This diff is collapsed.
......@@ -13,7 +13,7 @@ class TestMUC(FullStackTest):
extra_test_file)