Attachment 5034 Details for Bug 30258

[patch] Proof of concept for a threading decorator

30258.patch (text/plain), 13.13 KB, created by Dirk Wiesenthal on 2013-02-01 20:01 CET

(hide)

Description:

Filename:

MIME Type:

Creator: Dirk Wiesenthal

Created: 2013-02-01 20:01 CET

Size: 13.13 KB

patch

obsolete

>Index: src/univention/management/console/modules/decorators.py
>===================================================================
>--- src/univention/management/console/modules/decorators.py	(Revision 38462)
>+++ src/univention/management/console/modules/decorators.py	(Arbeitskopie)
>@@ -53,6 +53,7 @@
> """
> 
> import inspect
>+from threading import Thread
> 
> from univention.lib.i18n import Translation
> _ = Translation( 'univention.management.console' ).translate
>@@ -258,12 +259,26 @@
> 	if function is None:
> 		return lambda f: simple_response(f, with_flavor)
> 
>+	# does it
>+	#   return value ?
>+	#   => just send the value to the frontend
>+	# or
>+	#   yield value ?
>+	#   => send the value to the frontend and
>+	#      finish the function in a thread
>+	function_yields = inspect.isgeneratorfunction(function)
>+
> 	# fake a generator function that yields whatever the original
> 	# function returned
> 	def _fake_func(self, iterator, *args):
> 		for args in iterator:
> 			break
>-		yield function(self, *args)
>+		if function_yields:
>+			for res in function(self, *args):
>+				yield res
>+		else:
>+			res = function(self, *args)
>+			yield res
> 	copy_function_meta_data(function, _fake_func, copy_arg_inspect=True)
> 	# fake another variable name
> 	# the name is not important as it is removed from the list while
>@@ -279,8 +294,15 @@
> 
> 		# fake a multi_request
> 		request.options = [request.options]
>-		result = _multi_response(self, request)
>-		self.finished(request.id, result[0])
>+		gen = _multi_response(self, request)
>+		result = gen.next()
>+		self.finished(request.id, result)
>+		if function_yields:
>+			def _run_to_end(local_gen):
>+				for result in local_gen:
>+					pass
>+			thread = Thread(target=_run_to_end, args=[gen])
>+			thread.start()
> 
> 	copy_function_meta_data(function, _response)
> 	return _response
>@@ -329,7 +351,7 @@
> 		return lambda f: multi_response(f, with_flavor, single_values)
> 	response_func = _eval_simple_decorated_function(function, with_flavor, single_values)
> 	def _response(self, request):
>-		result = response_func(self, request)
>+		result = list(response_func(self, request))
> 		self.finished(request.id, result)
> 	copy_function_meta_data(function, _response)
> 	return _response
>@@ -375,16 +397,10 @@
> 							# check for required arguments (those without default)
> 							raise UMC_OptionMissing(arg)
> 
>-		
> 		# checked for required arguments, set default... now run!
>-		result = []
>-
> 		iterator = RequestOptionsIterator(request.options, arguments, single_values)
> 		nones = [None] * len(arguments)
>-		for res in function(self, iterator, *nones):
>-			result.append(res)
>-
>-		return result
>+		return function(self, iterator, *nones)
> 	return _response
> 
> class RequestOptionsIterator(object):
>@@ -439,31 +455,34 @@
> 	# copy __module__, otherwise it would be "univention.management.console.modules.decorators"
> 	new_function.__module__ = original_function.__module__
> 
>-def log(function=None, sensitives=None, customs=None, single_values=False):
>-	'''Log decorator to be used with
>-	:func:`simple_response`::
>+def log(function=None, sensitives=None, customs=None, single_values=False, logger=None, simple_threaded=False):
>+	'''Log decorator to be used with :func:`simple_response`.
> 
>+	You can specify any function to handle the log information. By default
>+	it is handled by MODULE.info, but you can pass MODULE.process (already
>+	logs when *umc/module/debug/level* is set to 2) as well as any other
>+	function that can handle strings::
>+
> 	 @simple_response
> 	 @log
> 	 def my_func(self, var1, var2):
> 	     return "%s__%s" % (var1, var2)
> 
>-	The above example will write two lines into the logfile for the
>-	module (given that the the UCR variable *umc/module/debug/level*
>-	is set to at least 3)::
>+	The above example will write two lines into the logfile for the module
>+	(given that the the UCR variable *umc/module/debug/level* is set to at
>+	least 3)::
> 
> 	 <date>  MODULE      ( INFO    ) : my_func got: var1='value1', var2='value2'
> 	 <date>  MODULE      ( INFO    ) : my_func returned: 'value1__value2'
> 
> 	The variable names are ordered by appearance and hold the values that
> 	are actually going to be passed to the function (i.e. after they were
>-	:func:`sanitize` 'd or set to their default value).
>-	You may specify the names of sensitive arguments that should not
>-	show up in log files and custom functions that can alter the
>-	representation of a certain variable's values (useful for non-standard
>-	datatypes like regular expressions - you may have used a
>-	:class:`~univention.management.console.modules.sanitizers.PatternSanitizer`
>-	)::
>+	:func:`sanitize` 'd or set to their default value). You may specify the
>+	names of sensitive arguments that should not show up in log files and
>+	custom functions that can alter the representation of a certain
>+	variable's values (useful for non-standard datatypes like regular
>+	expressions - you may have used a
>+	:class:`~univention.management.console.modules.sanitizers.PatternSanitizer`)::
> 
> 	 @sanitize(pattern=PatternSanitizer())
> 	 @simple_reponse
>@@ -479,23 +498,25 @@
> 	The decorator also works with :func:`multi_response`::
> 
> 	 @multi_response
>-	 @log
>+	 @log(logger=MODULE.process) # logs to PROCESS, not INFO
> 	 def multi_my_func(self, var1, var2):
> 	     return "%s__%s" % (var1, var2)
> 
> 	This results in something like::
> 
>-	 <date>  MODULE      ( INFO    ) : multi_my_func got: [var1='value1', var2='value2'], [var1='value3', var2='value4']
>-	 <date>  MODULE      ( INFO    ) : multi_my_func returned: ['value1__value2', 'value3__value4']
>+	 <date>  MODULE      ( PROCESS ) : multi_my_func got: [var1='value1', var2='value2'], [var1='value3', var2='value4']
>+	 <date>  MODULE      ( PROCESS ) : multi_my_func returned: ['value1__value2', 'value3__value4']
> 	'''
> 	if function is None:
>-		return lambda f: log(f, sensitives, customs, single_values)
>+		return lambda f: log(f, sensitives, customs, single_values, logger, simple_threaded)
> 	if customs is None:
> 		customs = {}
> 	if sensitives is None:
> 		sensitives = []
> 	for sensitive in sensitives:
> 		customs[sensitive] = lambda x: '********'
>+	if logger is None:
>+		logger = MODULE.info
> 
> 	def _log(names, args):
> 		if single_values:
>@@ -507,31 +528,49 @@
> 	name = function.__name__
> 	# multi_response yields i.e. is generator function
> 	if inspect.isgeneratorfunction(function):
>-		# remove self, iterator
>-		names = names[2:]
>-		def _response(self, iterator, *args):
>-			arg_reprs = []
>-			for element in iterator:
>-				arg_repr = _log(names, element)
>+		# problem: simple_response can also yield
>+		if simple_threaded:
>+			# remove self
>+			names = names[1:]
>+			def _response_simple_threaded(self, *args):
>+				arg_repr = _log(names, args)
> 				if arg_repr:
>-					arg_reprs.append(arg_repr)
>-			if arg_reprs:
>-				MODULE.info('%s got: [%s]' % (name, '], ['.join(', '.join(arg_repr) for arg_repr in arg_reprs)))
>-			result = []
>-			for res in function(self, iterator, *args):
>-				result.append(res)
>+					logger('%s got: %s' % (name, ', '.join(arg_repr)))
>+				result = function(self, *args)
>+				res = result.next()
>+				logger('%s returned: %r' % (name, res))
> 				yield res
>-			MODULE.info('%s returned: %r' % (name, result))
>+				for res in result:
>+					yield res
>+			_response = _response_simple_threaded
>+		else:
>+			# remove self, iterator
>+			names = names[2:]
>+			def _response_multiple(self, iterator, *args):
>+				arg_reprs = []
>+				for element in iterator:
>+					arg_repr = _log(names, element)
>+					if arg_repr:
>+						arg_reprs.append(arg_repr)
>+				if arg_reprs:
>+					logger('%s got: [%s]' % (name, '], ['.join(', '.join(arg_repr) for arg_repr in arg_reprs)))
>+				result = []
>+				for res in function(self, iterator, *args):
>+					result.append(res)
>+					yield res
>+				logger('%s returned: %r' % (name, result))
>+			_response = _response_multiple
> 	else:
> 		# remove self
> 		names = names[1:]
>-		def _response(self, *args):
>+		def _response_simple(self, *args):
> 			arg_repr = _log(names, args)
> 			if arg_repr:
>-				MODULE.info('%s got: %s' % (name, ', '.join(arg_repr)))
>+				logger('%s got: %s' % (name, ', '.join(arg_repr)))
> 			result = function(self, *args)
>-			MODULE.info('%s returned: %r' % (name, result))
>+			logger('%s returned: %r' % (name, result))
> 			return result
>+		_response = _response_simple
> 	copy_function_meta_data(function, _response, copy_arg_inspect=True)
> 	return _response
> 
>Index: umc/python/appcenter/__init__.py
>===================================================================
>--- umc/python/appcenter/__init__.py	(Revision 38793)
>+++ umc/python/appcenter/__init__.py	(Arbeitskopie)
>@@ -38,14 +38,10 @@
> import urllib2
> import re
> 
>-# related third party
>-import notifier
>-import notifier.threads
>-
> # univention
> from univention.lib.package_manager import PackageManager, LockError
> from univention.management.console.log import MODULE
>-from univention.management.console.modules.decorators import simple_response, sanitize, sanitize_list, multi_response
>+from univention.management.console.modules.decorators import simple_response, sanitize, sanitize_list, multi_response, log
> from univention.management.console.modules.sanitizers import PatternSanitizer, MappingSanitizer, DictSanitizer, StringSanitizer, ChoicesSanitizer, ListSanitizer, EmailSanitizer, BooleanSanitizer
> from univention.updater import UniventionUpdater
> from univention.updater.errors import ConfigurationError
>@@ -150,11 +146,11 @@
> 			application=StringSanitizer(minimum=1, required=True),
> 			force=BooleanSanitizer()
> 		)
>-	def invoke(self, request):
>-		function = request.options.get('function')
>-		application_id = request.options.get('application')
>+	@simple_response
>+	@log(logger=MODULE.process, simple_threaded=True)
>+	def invoke(self, function, application, force):
>+		application_id = application
> 		application = Application.find(application_id)
>-		force = request.options.get('force')
> 		try:
> 			# make sure that the application cane be installed/updated
> 			can_continue = True
>@@ -184,23 +180,18 @@
> 				result['remove'] = application.uninstall_dry_run(self.package_manager)
> 				can_continue = False
> 			result['can_continue'] = can_continue
>-			self.finished(request.id, result)
> 
>+			yield result
>+
> 			if can_continue:
>-				def _thread(module, application, function):
>-					with module.package_manager.locked(reset_status=True, set_finished=True):
>-						with module.package_manager.no_umc_restart(exclude_apache=True):
>-							if function in ('install', 'update'):
>-								# dont have to add component: already added during dry_run
>-								return application.install(module.package_manager, module.component_manager, add_component=False, send_as=function)
>-							else:
>-								return application.uninstall(module.package_manager, module.component_manager)
>-				def _finished(thread, result):
>-					if isinstance(result, BaseException):
>-						MODULE.warn('Exception during %s %s: %s' % (function, application_id, str(result)))
>-				thread = notifier.threads.Simple('invoke',
>-					notifier.Callback(_thread, self, application, function), _finished)
>-				thread.run()
>+				with self.package_manager.locked(reset_status=True, set_finished=True):
>+					with self.package_manager.no_umc_restart(exclude_apache=True):
>+						if function in ('install', 'update'):
>+							# dont have to add component: already added during dry_run
>+							application.install(self.package_manager, self.component_manager, add_component=False, send_as=function)
>+						else:
>+							application.uninstall(self.package_manager, self.component_manager)
>+
> 		except LockError:
> 			# make it thread safe: another process started a package manager
> 			# this module instance already has a running package manager
>@@ -282,30 +273,23 @@
> 			}, required=True),
> 		packages=ListSanitizer(StringSanitizer(minimum=1), required=True)
> 		)
>-	def packages_invoke(self, request):
>+	@simple_response
>+	@log(logger=MODULE.process, simple_threaded=True)
>+	def packages_invoke(self, packages, function):
> 		""" executes an installer action """
>-		packages = request.options.get('packages')
>-		function = request.options.get('function')
> 
> 		try:
> 			with self.package_manager.locked(reset_status=True):
> 				not_found = [pkg_name for pkg_name in packages if self.package_manager.get_package(pkg_name) is None]
>-				self.finished(request.id, {'not_found' : not_found})
>+				yield {'not_found' : not_found}
> 
> 				if not not_found:
>-					def _thread(package_manager, function, packages):
>-						with package_manager.locked(set_finished=True):
>-							with package_manager.no_umc_restart():
>-								if function == 'install':
>-									package_manager.install(*packages)
>-								else:
>-									package_manager.uninstall(*packages)
>-					def _finished(thread, result):
>-						if isinstance(result, BaseException):
>-							MODULE.warn('Exception during %s %s: %r' % (function, packages, str(result)))
>-					thread = notifier.threads.Simple('invoke', 
>-						notifier.Callback(_thread, self.package_manager, function, packages), _finished)
>-					thread.run()
>+					with self.package_manager.locked(set_finished=True):
>+						with self.package_manager.no_umc_restart():
>+							if function == 'install':
>+								self.package_manager.install(*packages)
>+							else:
>+								self.package_manager.uninstall(*packages)
> 		except LockError:
> 			# make it thread safe: another process started a package manager
> 			# this module instance already has a running package manager

Actions: View | Diff

Attachments on bug 30258: 5034