Commit 83a44f3c authored by Garrett Regier's avatar Garrett Regier

Use Python to implement the plugin loader's logic

This allows us to avoid the CPython API and have a
more understandable implementation.

https://bugzilla.gnome.org/show_bug.cgi?id=742349
parent b95aebb3
This diff is collapsed.
......@@ -27,21 +27,54 @@
#include <gio/gio.h>
/* _POSIX_C_SOURCE is defined in Python.h and in limits.h included by
* glib-object.h, so we unset it here to avoid a warning. Yep, that's bad.
*/
#undef _POSIX_C_SOURCE
#include <Python.h>
typedef PyObject _PeasPythonInternal;
static PyObject *FailedError = NULL;
static PyObject *
failed_fn (PyObject *self,
PyObject *args)
{
const char *msg;
if (!PyArg_ParseTuple (args, "s:Hooks.failed", &msg))
return NULL;
g_warning ("%s", msg);
/* peas_python_internal_call() knows to check for this exception */
PyErr_SetObject (FailedError, NULL);
return NULL;
}
static PyMethodDef failed_method_def = {
"failed", (PyCFunction) failed_fn, METH_VARARGS | METH_STATIC,
"Prints warning and raises an Exception"
};
PeasPythonInternal *
peas_python_internal_new (void)
peas_python_internal_new (gboolean already_initialized)
{
GBytes *internal_python;
PyObject *builtins_module, *code, *globals, *result, *internal;
const gchar *prgname;
PeasPythonInternal *internal = NULL;
PyObject *builtins_module, *globals;
PyObject *code = NULL, *module = NULL;
PyObject *result = NULL, *failed_method = NULL;
#define goto_error_if_failed(cond) \
G_STMT_START { \
if (G_UNLIKELY (!(cond))) \
{ \
g_warn_if_fail (cond); \
goto error; \
} \
} G_STMT_END
prgname = g_get_prgname ();
prgname = prgname == NULL ? "" : prgname;
#if PY_MAJOR_VERSION < 3
builtins_module = PyImport_ImportModule ("__builtin__");
......@@ -49,7 +82,7 @@ peas_python_internal_new (void)
builtins_module = PyImport_ImportModule ("builtins");
#endif
g_return_val_if_fail (builtins_module != NULL, NULL);
goto_error_if_failed (builtins_module != NULL);
/* We don't use the byte-compiled Python source
* because glib-compile-resources cannot output
......@@ -57,7 +90,6 @@ peas_python_internal_new (void)
*
* https://bugzilla.gnome.org/show_bug.cgi?id=673101
*/
internal_python = g_resources_lookup_data ("/org/gnome/libpeas/loaders/"
#if PY_MAJOR_VERSION < 3
"python/"
......@@ -67,72 +99,138 @@ peas_python_internal_new (void)
"internal.py",
G_RESOURCE_LOOKUP_FLAGS_NONE,
NULL);
g_return_val_if_fail (internal_python != NULL, NULL);
goto_error_if_failed (internal_python != NULL);
/* Compile it manually so the filename is available */
code = Py_CompileString (g_bytes_get_data (internal_python, NULL),
"peas-python-internal.py",
Py_file_input);
g_bytes_unref (internal_python);
g_return_val_if_fail (code != NULL, NULL);
globals = PyDict_New ();
if (globals == NULL)
{
Py_DECREF (code);
g_return_val_if_fail (globals != NULL, NULL);
}
if (PyDict_SetItemString (globals, "__builtins__",
PyModule_GetDict (builtins_module)) != 0)
{
Py_DECREF (globals);
Py_DECREF (code);
return NULL;
}
goto_error_if_failed (code != NULL);
module = PyModule_New ("libpeas-internal");
goto_error_if_failed (module != NULL);
goto_error_if_failed (PyModule_AddObject (module, "__builtins__",
builtins_module) == 0);
goto_error_if_failed (PyModule_AddObject (module, "ALREADY_INITIALIZED",
already_initialized ?
Py_True : Py_False) == 0);
goto_error_if_failed (PyModule_AddStringConstant (module, "PRGNAME",
prgname) == 0);
goto_error_if_failed (PyModule_AddStringMacro (module,
PEAS_PYEXECDIR) == 0);
goto_error_if_failed (PyModule_AddStringMacro (module,
GETTEXT_PACKAGE) == 0);
goto_error_if_failed (PyModule_AddStringMacro (module,
PEAS_LOCALEDIR) == 0);
globals = PyModule_GetDict (module);
result = PyEval_EvalCode ((gpointer) code, globals, globals);
Py_XDECREF (result);
Py_DECREF (code);
if (PyErr_Occurred ())
{
Py_DECREF (globals);
return NULL;
g_warning ("Failed to run internal Python code");
goto error;
}
internal = PyDict_GetItemString (globals, "hooks");
Py_XINCREF (internal);
Py_DECREF (globals);
result = PyDict_GetItemString (globals, "hooks");
goto_error_if_failed (result != NULL);
goto_error_if_failed (PyObject_SetAttrString (result,
"__internal_module__",
module) == 0);
FailedError = PyDict_GetItemString (globals, "FailedError");
goto_error_if_failed (FailedError != NULL);
failed_method = PyCFunction_NewEx (&failed_method_def, NULL, module);
goto_error_if_failed (failed_method != NULL);
goto_error_if_failed (PyObject_SetAttrString (result, "failed",
failed_method) == 0);
g_return_val_if_fail (internal != NULL, NULL);
return (PeasPythonInternal *) internal;
internal = (PeasPythonInternal *) result;
#undef goto_error_if_failed
error:
if (internal == NULL)
Py_XDECREF (result);
Py_XDECREF (failed_method);
Py_XDECREF (module);
Py_XDECREF (code);
g_clear_pointer (&internal_python, g_bytes_unref);
return internal;
}
/* NOTE: This must be called with the GIL held */
void
peas_python_internal_free (PeasPythonInternal *internal)
{
PyObject *internal_ = (PyObject *) internal;
peas_python_internal_call (internal, "exit");
peas_python_internal_call (internal, "exit", NULL, NULL);
Py_DECREF (internal_);
}
/* NOTE: This must be called with the GIL held */
void
PyObject *
peas_python_internal_call (PeasPythonInternal *internal,
const gchar *name)
const gchar *name,
PyTypeObject *return_type,
const gchar *format,
...)
{
PyObject *internal_ = (PyObject *) internal;
PyObject *result;
PyObject *callable, *args;
PyObject *result = NULL;
va_list var_args;
result = PyObject_CallMethod (internal_, (gchar *) name, NULL);
Py_XDECREF (result);
/* The PyTypeObject for Py_None is not exposed directly */
if (return_type == NULL)
return_type = Py_None->ob_type;
callable = PyObject_GetAttrString (internal_, name);
g_return_val_if_fail (callable != NULL, NULL);
va_start (var_args, format);
args = Py_VaBuildValue (format == NULL ? "()" : format, var_args);
va_end (var_args);
if (args != NULL)
{
result = PyObject_CallObject (callable, args);
Py_DECREF (args);
}
if (PyErr_Occurred ())
PyErr_Print ();
}
{
/* Raised by failed_fn() to prevent printing the exception */
if (PyErr_ExceptionMatches (FailedError))
{
PyErr_Clear ();
}
else
{
g_warning ("Failed to run internal Python hook '%s'", name);
PyErr_Print ();
}
return NULL;
}
/* We always allow a None result */
if (result == Py_None)
{
Py_CLEAR (result);
}
else if (!PyObject_TypeCheck (result, return_type))
{
g_warning ("Failed to run internal Python hook '%s': ", name);
Py_CLEAR (result);
}
return result;
}
......@@ -24,16 +24,26 @@
#include <glib.h>
/* _POSIX_C_SOURCE is defined in Python.h and in limits.h included by
* glib-object.h, so we unset it here to avoid a warning. Yep, that's bad.
*/
#undef _POSIX_C_SOURCE
#include <Python.h>
G_BEGIN_DECLS
typedef struct _PeasPythonInternal PeasPythonInternal;
PeasPythonInternal *
peas_python_internal_new (void);
peas_python_internal_new (gboolean already_initialized);
void peas_python_internal_free (PeasPythonInternal *internal);
void peas_python_internal_call (PeasPythonInternal *internal,
const gchar *name);
PyObject *
peas_python_internal_call (PeasPythonInternal *internal,
const gchar *name,
PyTypeObject *return_type,
const gchar *format,
...);
G_END_DECLS
......
# -*- coding: utf-8 -*-
# Copyright (C) 2014 - Garrett Regier
# Copyright (C) 2014-2015 - Garrett Regier
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library General Public License as published by
......@@ -17,83 +17,204 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
import cProfile
import gc
import gettext
import importlib
import os
import pstats
import signal
import sys
import threading
import weakref
import traceback
from gi.repository import GLib, GObject
# Derive from something not normally caught
class FailedError(BaseException):
pass
class Hooks(object):
def __init__(self):
self.profiling_enabled = os.getenv('PEAS_PYTHON_PROFILE') is not None
if not self.profiling_enabled:
return
if not ALREADY_INITIALIZED:
int_handler = signal.getsignal(signal.SIGINT)
sort = os.getenv('PEAS_PYTHON_PROFILE', default='time')
self.stat_sort = sort.split(';')
# Use the default handler instead of raising KeyboardInterrupt
if int_handler == signal.default_int_handler:
signal.signal(signal.SIGINT, signal.SIG_DFL)
self.stats = None
self.stats_lock = threading.Lock()
# See PySys_SetArgvEx()
sys.argv = [PRGNAME]
sys.path.pop(0)
self.thread_refs = []
self.thread_local = threading.local()
sys.path.insert(0, PEAS_PYEXECDIR)
gettext.install(GETTEXT_PACKAGE, PEAS_LOCALEDIR)
threading.setprofile(self.init_thread)
self.__idle_gc = 0
self.__module_cache = {}
self.__extension_cache = {}
self.profile = cProfile.Profile()
self.profile.enable()
@staticmethod
def failed():
# This is implemented by the plugin loader
raise NotImplementedError('Hooks.failed()')
def add_stats(self, profile):
profile.disable()
def load(self, filename, module_dir, module_name):
try:
return self.__module_cache[filename]
with self.stats_lock:
if self.stats is None:
self.stats = pstats.Stats(profile)
except KeyError:
pass
else:
self.stats.add(profile)
if module_name in sys.modules:
self.__module_cache[filename] = None
self.failed("Error loading plugin '%s': "
"module name '%s' has already been used" %
(filename, module_name))
def init_thread(self, *unused):
# Only call once per thread
sys.setprofile(None)
if module_dir not in sys.path:
sys.path.insert(0, module_dir)
thread_profile = cProfile.Profile()
try:
module = importlib.import_module(module_name)
def thread_finished(thread_ref):
self.add_stats(thread_profile)
except:
module = None
self.failed("Error importing plugin '%s':\n%s" %
(module_name, traceback.format_exc()))
self.thread_refs.remove(thread_ref)
else:
self.__extension_cache[module] = {}
# Need something to weakref, the
# current thread does not support it
thread_ref = set()
self.thread_local.ref = thread_ref
finally:
self.__module_cache[filename] = module
self.thread_refs.append(weakref.ref(thread_ref, thread_finished))
return module
# Only enable the profile at the end
thread_profile.enable()
def find_extension_type(self, gtype, module):
module_gtypes = self.__extension_cache[module]
def all_plugins_unloaded(self):
if not self.profiling_enabled:
return
try:
return module_gtypes[gtype]
except KeyError:
pass
for key in module.__dict__:
value = getattr(module, key)
try:
value_gtype = value.__gtype__
except AttributeError:
continue
if GObject.type_is_a(value_gtype, gtype):
module_gtypes[gtype] = value
return value
module_gtypes[gtype] = None
return None
self.add_stats(self.profile)
def __run_gc(self):
gc.collect()
with self.stats_lock:
self.stats.strip_dirs().sort_stats(*self.stat_sort).print_stats()
self.__idle_gc = 0
return False
# Need to create a new profile to avoid adding the stats twice
self.profile = cProfile.Profile()
self.profile.enable()
def garbage_collect(self):
# We run the GC right now and we schedule
# a further collection in the main loop
gc.collect()
if self.__idle_gc == 0:
self.__idle_gc = GLib.idle_add(self.__run_gc)
GLib.source_set_name_by_id(self.__idle_gc, '[libpeas] run_gc')
def all_plugins_unloaded(self):
pass
def exit(self):
if not self.profiling_enabled:
return
gc.collect()
if self.__idle_gc != 0:
GLib.source_remove(self.__idle_gc)
if os.getenv('PEAS_PYTHON_PROFILE') is not None:
import cProfile
import pstats
import threading
import weakref
class Hooks(Hooks):
def __init__(self):
super(Hooks, self).__init__()
sort = os.getenv('PEAS_PYTHON_PROFILE', default='time')
self.__stat_sort = sort.split(';')
self.__stats = None
self.__stats_lock = threading.Lock()
self.__thread_refs = []
self.__thread_local = threading.local()
threading.setprofile(self.__init_thread)
self.__profile = cProfile.Profile()
self.__profile.enable()
def __add_stats(self, profile):
profile.disable()
with self.__stats_lock:
if self.__stats is None:
self.__stats = pstats.Stats(profile)
else:
self.__stats.add(profile)
def __init_thread(self, *unused):
# Only call once per thread
sys.setprofile(None)
thread_profile = cProfile.Profile()
def thread_finished(thread_ref):
self.__add_stats(thread_profile)
self.__thread_refs.remove(thread_ref)
# Need something to weakref, the
# current thread does not support it
thread_ref = set()
self.__thread_local.ref = thread_ref
self.__thread_refs.append(weakref.ref(thread_ref,
thread_finished))
# Only enable the profile at the end
thread_profile.enable()
def all_plugins_unloaded(self):
super(Hooks, self).all_plugins_unloaded()
self.__add_stats(self.__profile)
with self.__stats_lock:
stats = self.__stats.strip_dirs()
stats.sort_stats(*self.__stat_sort)
stats.print_stats()
# Need to create a new profile to avoid adding the stats twice
self.__profile = cProfile.Profile()
self.__profile.enable()
def exit(self):
super(Hooks, self).exit()
self.profile.disable()
self.__profile.disable()
hooks = Hooks()
......
......@@ -107,20 +107,12 @@ test_extension_py_activatable_subject_refcount (PeasEngine *engine,
}
static void
test_extension_py_nonexistent (void)
{
g_test_trap_subprocess (EXTENSION_TEST_NAME (PY_LOADER,
"nonexistent/subprocess"),
0, 0);
g_test_trap_assert_passed ();
g_test_trap_assert_stderr ("*ImportError*");
}
static void
test_extension_py_nonexistent_subprocess (PeasEngine *engine)
test_extension_py_nonexistent (PeasEngine *engine)
{
PeasPluginInfo *info;
testing_util_push_log_hook ("Error importing plugin 'extension-"
PY_LOADER_STR "-nonexistent'*");
testing_util_push_log_hook ("Error loading plugin 'extension-"
PY_LOADER_STR "-nonexistent'");
......@@ -248,9 +240,7 @@ main (int argc,
EXTENSION_TEST (PY_LOADER, "activatable-subject-refcount",
activatable_subject_refcount);
EXTENSION_TEST_FUNC (PY_LOADER, "nonexistent", nonexistent);
EXTENSION_TEST (PY_LOADER, "nonexistent/subprocess",
nonexistent_subprocess);
EXTENSION_TEST (PY_LOADER, "nonexistent", nonexistent);
EXTENSION_TEST_FUNC (PY_LOADER, "already-initialized", already_initialized);
EXTENSION_TEST_FUNC (PY_LOADER, "already-initialized/subprocess",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment