Merge lp:~zeitgeist/zeitgeist/autoload-extensions into lp:zeitgeist/0.1

[Mikkel Kamstrup Erlandsen]

THis branch implements automatic loading of extensions found in the extensions/ directory. You can still override this via the ZEITGEIST_{DEFAULT,EXTRA}_EXTENSIONS envvars.

It also installs zeitgeist-daemon.pc which defines a 'extensiondir' variable 3rd party extension developers can use to detect the path in which to install extensions.

[Markus Korn (thekorn)]

First of all: I don't zhave enough autotools knowlege to understand what the non-python part of this patch is all about, but some test installations show that it seems to work as expected.

This is why I only had a closer look at the python part, it is looking perfect to me, BUT I have a few minor comments:

dict.get(<key>) always returns None if <key> is not found in dict, this is why os.environ("ENVVAR_NAME", None) should be simplified to os.environ("ENVVAR_NAME")

And I've a second, not too serious comment: You are doing to much C stuff! - this explains to me the ugly extra spaces you are adding to the code ;)
Some examples:
* extensions = _scan_extensions () or _zg = __import__ ("_zeitgeist.engine.extensions." + mod)
* log.debug("Searching for extensions in: %s" % config.extensiondir) or modules = filter(lambda m : m.endswith(".py"), os.listdir(config.extensiondir))

Other that that, big Approval.

thnks for your work,

markus

review: Approve (python parts)

[lp:~zeitgeist/zeitgeist/autoload-extensions]

1506. By Mikkel Kamstrup Erlandsen <kamstrup@hardback> on 2010-07-01

Stylistic updates to reflect Markus' comments

=== modified file 'Makefile.am'
--- Makefile.am	2009-12-08 14:08:44 +0000
+++ Makefile.am	2010-07-01 07:47:23 +0000
@@ -16,7 +16,8 @@
 	COPYRIGHT \
 	NEWS \
 	zeitgeist-daemon.py \
-	zeitgeist-datahub.py
+	zeitgeist-datahub.py \
+	zeitgeist-daemon.pc.in
 
 DISTCLEANFILES = \
 	intltool-extract \
@@ -27,6 +28,9 @@
 	zeitgeist-daemon \
 	zeitgeist-datahub
 
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = zeitgeist-daemon.pc
+
 zeitgeist-daemon: zeitgeist-daemon.py
 	sed \
 		-e "s!\/usr\/bin\/env python!$(PYTHON)!" \

=== modified file '_zeitgeist/Makefile.am'
--- _zeitgeist/Makefile.am	2010-05-16 15:34:50 +0000
+++ _zeitgeist/Makefile.am	2010-07-01 07:47:23 +0000
@@ -1,6 +1,6 @@
 SUBDIRS = engine loggers
 
-appdir = $(prefix)/share/zeitgeist/_zeitgeist/
+appdir = $(datadir)/zeitgeist/_zeitgeist/
 
 app_PYTHON = \
 	__init__.py \

=== modified file '_zeitgeist/engine/Makefile.am'
--- _zeitgeist/engine/Makefile.am	2010-06-08 10:21:27 +0000
+++ _zeitgeist/engine/Makefile.am	2010-07-01 07:47:23 +0000
@@ -1,6 +1,6 @@
 SUBDIRS = extensions upgrades
 
-appdir = $(prefix)/share/zeitgeist/_zeitgeist/engine/
+appdir = $(datadir)/zeitgeist/_zeitgeist/engine/
 
 app_PYTHON = \
 	__init__.py \

=== modified file '_zeitgeist/engine/__init__.py'
--- _zeitgeist/engine/__init__.py	2010-05-27 19:51:09 +0000
+++ _zeitgeist/engine/__init__.py	2010-07-01 07:47:23 +0000
@@ -42,31 +42,6 @@
 		import main # _zeitgeist.engine.main
 		_engine = main.ZeitgeistEngine()
 	return _engine
-	
-def _get_extensions():
-	"""looks at the `ZEITGEIST_DEFAULT_EXTENSIONS` environment variable
-	to find the extensions which should be loaded on daemon startup, if
-	this variable is not set the `Blacklist` and the `DataSourceRegistry`
-	extension will be loaded. If this variable is set to an empty string
-	no extensions are loaded by default.
-	To load an extra set of extensions define the `ZEITGEIST_EXTRA_EXTENSIONS`
-	variable.
-	The format of these variables should just be a no-space comma
-	separated list of module.class names"""
-	default_extensions = os.environ.get("ZEITGEIST_DEFAULT_EXTENSIONS", None)
-	if default_extensions is not None:
-		extensions = default_extensions.split(",")
-	else:
-		extensions = [
-		"_zeitgeist.engine.extensions.blacklist.Blacklist",
-		"_zeitgeist.engine.extensions.datasource_registry.DataSourceRegistry",
-		]
-	extra_extensions = os.environ.get("ZEITGEIST_EXTRA_EXTENSIONS", None)
-	if extra_extensions is not None:
-		extensions += extra_extensions.split(",")
-	extensions = filter(None, extensions)
-	log.debug("daemon is configured to run with these extensions: %r" %extensions)
-	return extensions
 
 class _Constants:
 	# Directories
@@ -79,9 +54,6 @@
 	DBUS_INTERFACE = ZeitgeistDBusInterface.INTERFACE_NAME
 	SIG_EVENT = "asaasay"
 	
-	# Extensions
-	DEFAULT_EXTENSIONS = _get_extensions()
-	
 	# Required version of DB schema
 	CORE_SCHEMA="core"
 	CORE_SCHEMA_VERSION = 1

=== modified file '_zeitgeist/engine/extension.py'
--- _zeitgeist/engine/extension.py	2010-07-01 07:38:58 +0000
+++ _zeitgeist/engine/extension.py	2010-07-01 07:47:23 +0000
@@ -3,6 +3,7 @@
 # Zeitgeist
 #
 # Copyright © 2009 Markus Korn <[e-mail]>
+# Copyright © 2010 Mikkel Kamstrup Erlandsen <[e-mail]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -17,12 +18,15 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import logging
 import weakref # avoid circular references as they confuse garbage collection
 
 logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("zeitgeist.extension")
 
+import zeitgeist
+
 def safe_issubclass(obj, cls):
 	try:
 		return issubclass(obj, cls)
@@ -98,7 +102,48 @@
 		pass
 
 
-def load_class(path):
+def get_extensions():
+	"""looks at the `ZEITGEIST_DEFAULT_EXTENSIONS` environment variable
+	to find the extensions which should be loaded on daemon startup, if
+	this variable is not set the "extensiondir" variable of the configuration
+	will be scanned for .py files with classes extending Extension
+	If this variable is set to an empty string no extensions are loaded by
+	default.
+	To load an extra set of extensions define the `ZEITGEIST_EXTRA_EXTENSIONS`
+	variable.
+	The format of these variables should just be a no-space comma
+	separated list of module.class names"""
+	default_extensions = os.environ.get("ZEITGEIST_DEFAULT_EXTENSIONS")
+	if default_extensions is not None:
+		extensions = map(_load_class, default_extensions.split(","))
+	else:
+		extensions = _scan_extensions()
+	extra_extensions = os.environ.get("ZEITGEIST_EXTRA_EXTENSIONS")
+	if extra_extensions is not None:
+		extensions += map(_load_class, extra_extensions.split(","))
+	extensions = filter(None, extensions)
+	log.debug("daemon is configured to run with these extensions: %r" %extensions)
+	return extensions
+
+def _scan_extensions():
+	"""Look in zeitgeist._config.extensiondir for .py files and return
+	a list of classes with all the classes that extends the Extension class"""
+	config = zeitgeist._config
+	log.debug("Searching for extensions in: %s" % config.extensiondir)	
+	extensions = []
+	modules = filter(lambda m : m.endswith(".py"), os.listdir(config.extensiondir))	
+	modules = map(lambda m : m.rpartition(".")[0], modules)
+	for mod in modules:
+		_zg = __import__("_zeitgeist.engine.extensions." + mod)
+		ext = getattr(_zg.engine.extensions, mod)
+		for cls in dir(ext):
+			cls = getattr(ext, cls)
+			if safe_issubclass(cls, Extension) and not cls is Extension:
+				extensions.append(cls)
+	return extensions
+			
+
+def _load_class(path):
 	"""
 	Load and return a class from a fully qualified string.
 	Fx. "_zeitgeist.engine.extensions.myext.MyClass"

=== modified file '_zeitgeist/engine/extensions/Makefile.am'
--- _zeitgeist/engine/extensions/Makefile.am	2010-01-21 00:42:48 +0000
+++ _zeitgeist/engine/extensions/Makefile.am	2010-07-01 07:47:23 +0000
@@ -1,4 +1,4 @@
-appdir = $(prefix)/share/zeitgeist/_zeitgeist/engine/extensions
+appdir = $(datadir)/zeitgeist/_zeitgeist/engine/extensions
 
 app_PYTHON = \
 	__init__.py \

=== modified file '_zeitgeist/engine/main.py'
--- _zeitgeist/engine/main.py	2010-07-01 07:34:26 +0000
+++ _zeitgeist/engine/main.py	2010-07-01 07:47:23 +0000
@@ -34,7 +34,7 @@
 from zeitgeist.datamodel import Event as OrigEvent, StorageState, TimeRange, \
 	ResultType, get_timestamp_for_now, Interpretation, Symbol, NEGATION_OPERATOR, WILDCARD
 from _zeitgeist.engine.datamodel import Event, Subject
-from _zeitgeist.engine.extension import ExtensionsCollection, load_class
+from _zeitgeist.engine.extension import ExtensionsCollection, get_extensions
 from _zeitgeist.engine import constants
 from _zeitgeist.engine.sql import get_default_cursor, unset_cursor, \
 	TableLookup, WhereClause
@@ -116,9 +116,9 @@
 			raise RuntimeError("old database version")
 		
 		# Load extensions
-		default_extensions = map(load_class, constants.DEFAULT_EXTENSIONS)
+		default_extensions = get_extensions()
 		self.__extensions = ExtensionsCollection(self,
-			defaults=default_extensions)
+		                                         defaults=default_extensions)
 		
 		self._interpretation = TableLookup(cursor, "interpretation")
 		self._manifestation = TableLookup(cursor, "manifestation")

=== modified file '_zeitgeist/engine/upgrades/Makefile.am'
--- _zeitgeist/engine/upgrades/Makefile.am	2010-06-08 10:21:27 +0000
+++ _zeitgeist/engine/upgrades/Makefile.am	2010-07-01 07:47:23 +0000
@@ -1,4 +1,4 @@
-appdir = $(prefix)/share/zeitgeist/_zeitgeist/engine/upgrades
+appdir = $(datadir)/zeitgeist/_zeitgeist/engine/upgrades
 
 app_PYTHON = \
 	__init__.py \

=== modified file '_zeitgeist/loggers/Makefile.am'
--- _zeitgeist/loggers/Makefile.am	2009-07-11 23:20:37 +0000
+++ _zeitgeist/loggers/Makefile.am	2010-07-01 07:47:23 +0000
@@ -1,6 +1,6 @@
 SUBDIRS = datasources
 
-appdir = $(prefix)/share/zeitgeist/_zeitgeist/loggers/
+appdir = $(datadir)/zeitgeist/_zeitgeist/loggers/
 
 app_PYTHON = \
 	__init__.py \

=== modified file '_zeitgeist/loggers/datasources/Makefile.am'
--- _zeitgeist/loggers/datasources/Makefile.am	2009-11-25 17:01:07 +0000
+++ _zeitgeist/loggers/datasources/Makefile.am	2010-07-01 07:47:23 +0000
@@ -1,4 +1,4 @@
-appdir = $(prefix)/share/zeitgeist/_zeitgeist/loggers/datasources
+appdir = $(datadir)/zeitgeist/_zeitgeist/loggers/datasources
 
 app_PYTHON = \
 	__init__.py \

=== added file 'acinclude.m4'
--- acinclude.m4	1970-01-01 00:00:00 +0000
+++ acinclude.m4	2010-07-01 07:47:23 +0000
@@ -0,0 +1,40 @@
+dnl AS_AC_EXPAND(VAR, CONFIGURE_VAR)
+dnl
+dnl example
+dnl AS_AC_EXPAND(SYSCONFDIR, $sysconfdir)
+dnl will set SYSCONFDIR to /usr/local/etc if prefix=/usr/local
+
+AC_DEFUN([AS_AC_EXPAND],
+[
+  EXP_VAR=[$1]
+  FROM_VAR=[$2]
+
+  dnl first expand prefix and exec_prefix if necessary
+  prefix_save=$prefix
+  exec_prefix_save=$exec_prefix
+
+  dnl if no prefix given, then use /usr/local, the default prefix
+  if test "x$prefix" = "xNONE"; then
+    prefix=$ac_default_prefix
+  fi
+  dnl if no exec_prefix given, then use prefix
+  if test "x$exec_prefix" = "xNONE"; then
+    exec_prefix=$prefix
+  fi
+
+  full_var="$FROM_VAR"
+  dnl loop until it doesn't change anymore
+  while true; do
+    new_full_var="`eval echo $full_var`"
+    if test "x$new_full_var"="x$full_var"; then break; fi
+    full_var=$new_full_var
+  done
+
+  dnl clean up
+  full_var=$new_full_var
+  AC_SUBST([$1], "$full_var")
+
+  dnl restore prefix and exec_prefix
+  prefix=$prefix_save
+  exec_prefix=$exec_prefix_save
+])

=== modified file 'configure.ac'
--- configure.ac	2010-06-21 10:22:38 +0000
+++ configure.ac	2010-07-01 07:47:23 +0000
@@ -13,12 +13,17 @@
 AM_GLIB_GNU_GETTEXT
 IT_PROG_INTLTOOL([0.35.0])
 
+# Expand variables needed for config.py
+AS_AC_EXPAND(DATADIR, $datarootdir)
+AC_SUBST(DATADIR)
+
 AC_CONFIG_FILES([
 	Makefile
 	doc/Makefile
 	extra/Makefile
 	extra/ontology/Makefile
 	po/Makefile.in
+	zeitgeist-daemon.pc	
 	zeitgeist/Makefile
 	_zeitgeist/Makefile
 	_zeitgeist/loggers/Makefile

=== added file 'zeitgeist-daemon.pc.in'
--- zeitgeist-daemon.pc.in	1970-01-01 00:00:00 +0000
+++ zeitgeist-daemon.pc.in	2010-07-01 07:47:23 +0000
@@ -0,0 +1,7 @@
+prefix=@prefix@
+extensiondir=@datadir@/zeitgeist/_zeitgeist/engine/extensions
+
+Name: @PACKAGE_NAME@
+Description: Zeitgeist Engine Extensions
+Version: @PACKAGE_VERSION@
+

=== modified file 'zeitgeist/__init__.py'
--- zeitgeist/__init__.py	2009-12-31 23:12:25 +0000
+++ zeitgeist/__init__.py	2010-07-01 07:47:23 +0000
@@ -23,6 +23,7 @@
         pkgdatadir = os.path.join(bindir, "data")
         privatepythondir = bindir
         datasourcedir = os.path.join(bindir, "_zeitgeist/loggers/datasources")
+        extensiondir = os.path.join(bindir, "_zeitgeist/engine/extensions")
         libdir = ""
         libexecdir = ""
         PACKAGE = "zeitgeist"

=== modified file 'zeitgeist/_config.py.in'
--- zeitgeist/_config.py.in	2009-12-30 19:03:24 +0000
+++ zeitgeist/_config.py.in	2010-07-01 07:47:23 +0000
@@ -4,7 +4,8 @@
 localedir = "@datadir@/locale"
 pkgdatadir = "@pkgdatadir@"
 privatepythondir = "@prefix@/share/zeitgeist"
-datasourcedir = "@prefix@/share/zeitgeist/_zeitgeist/loggers/datasources"
+datasourcedir = "@datadir@/zeitgeist/_zeitgeist/loggers/datasources"
+extensiondir = "@datadir@/zeitgeist/_zeitgeist/engine/extensions"
 libdir = "@libdir@"
 libexecdir = "@libexecdir@"
 PACKAGE = "@PACKAGE@"