Index: src/univention/management/console/modules/decorators.py =================================================================== --- src/univention/management/console/modules/decorators.py (Revision 38462) +++ src/univention/management/console/modules/decorators.py (Arbeitskopie) @@ -53,6 +53,7 @@ """ import inspect +from threading import Thread from univention.lib.i18n import Translation _ = Translation( 'univention.management.console' ).translate @@ -258,12 +259,26 @@ if function is None: return lambda f: simple_response(f, with_flavor) + # does it + # return value ? + # => just send the value to the frontend + # or + # yield value ? + # => send the value to the frontend and + # finish the function in a thread + function_yields = inspect.isgeneratorfunction(function) + # fake a generator function that yields whatever the original # function returned def _fake_func(self, iterator, *args): for args in iterator: break - yield function(self, *args) + if function_yields: + for res in function(self, *args): + yield res + else: + res = function(self, *args) + yield res copy_function_meta_data(function, _fake_func, copy_arg_inspect=True) # fake another variable name # the name is not important as it is removed from the list while @@ -279,8 +294,15 @@ # fake a multi_request request.options = [request.options] - result = _multi_response(self, request) - self.finished(request.id, result[0]) + gen = _multi_response(self, request) + result = gen.next() + self.finished(request.id, result) + if function_yields: + def _run_to_end(local_gen): + for result in local_gen: + pass + thread = Thread(target=_run_to_end, args=[gen]) + thread.start() copy_function_meta_data(function, _response) return _response @@ -329,7 +351,7 @@ return lambda f: multi_response(f, with_flavor, single_values) response_func = _eval_simple_decorated_function(function, with_flavor, single_values) def _response(self, request): - result = response_func(self, request) + result = list(response_func(self, request)) self.finished(request.id, result) copy_function_meta_data(function, _response) return _response @@ -375,16 +397,10 @@ # check for required arguments (those without default) raise UMC_OptionMissing(arg) - # checked for required arguments, set default... now run! - result = [] - iterator = RequestOptionsIterator(request.options, arguments, single_values) nones = [None] * len(arguments) - for res in function(self, iterator, *nones): - result.append(res) - - return result + return function(self, iterator, *nones) return _response class RequestOptionsIterator(object): @@ -439,31 +455,34 @@ # copy __module__, otherwise it would be "univention.management.console.modules.decorators" new_function.__module__ = original_function.__module__ -def log(function=None, sensitives=None, customs=None, single_values=False): - '''Log decorator to be used with - :func:`simple_response`:: +def log(function=None, sensitives=None, customs=None, single_values=False, logger=None, simple_threaded=False): + '''Log decorator to be used with :func:`simple_response`. + You can specify any function to handle the log information. By default + it is handled by MODULE.info, but you can pass MODULE.process (already + logs when *umc/module/debug/level* is set to 2) as well as any other + function that can handle strings:: + @simple_response @log def my_func(self, var1, var2): return "%s__%s" % (var1, var2) - The above example will write two lines into the logfile for the - module (given that the the UCR variable *umc/module/debug/level* - is set to at least 3):: + The above example will write two lines into the logfile for the module + (given that the the UCR variable *umc/module/debug/level* is set to at + least 3):: MODULE ( INFO ) : my_func got: var1='value1', var2='value2' MODULE ( INFO ) : my_func returned: 'value1__value2' The variable names are ordered by appearance and hold the values that are actually going to be passed to the function (i.e. after they were - :func:`sanitize` 'd or set to their default value). - You may specify the names of sensitive arguments that should not - show up in log files and custom functions that can alter the - representation of a certain variable's values (useful for non-standard - datatypes like regular expressions - you may have used a - :class:`~univention.management.console.modules.sanitizers.PatternSanitizer` - ):: + :func:`sanitize` 'd or set to their default value). You may specify the + names of sensitive arguments that should not show up in log files and + custom functions that can alter the representation of a certain + variable's values (useful for non-standard datatypes like regular + expressions - you may have used a + :class:`~univention.management.console.modules.sanitizers.PatternSanitizer`):: @sanitize(pattern=PatternSanitizer()) @simple_reponse @@ -479,23 +498,25 @@ The decorator also works with :func:`multi_response`:: @multi_response - @log + @log(logger=MODULE.process) # logs to PROCESS, not INFO def multi_my_func(self, var1, var2): return "%s__%s" % (var1, var2) This results in something like:: - MODULE ( INFO ) : multi_my_func got: [var1='value1', var2='value2'], [var1='value3', var2='value4'] - MODULE ( INFO ) : multi_my_func returned: ['value1__value2', 'value3__value4'] + MODULE ( PROCESS ) : multi_my_func got: [var1='value1', var2='value2'], [var1='value3', var2='value4'] + MODULE ( PROCESS ) : multi_my_func returned: ['value1__value2', 'value3__value4'] ''' if function is None: - return lambda f: log(f, sensitives, customs, single_values) + return lambda f: log(f, sensitives, customs, single_values, logger, simple_threaded) if customs is None: customs = {} if sensitives is None: sensitives = [] for sensitive in sensitives: customs[sensitive] = lambda x: '********' + if logger is None: + logger = MODULE.info def _log(names, args): if single_values: @@ -507,31 +528,49 @@ name = function.__name__ # multi_response yields i.e. is generator function if inspect.isgeneratorfunction(function): - # remove self, iterator - names = names[2:] - def _response(self, iterator, *args): - arg_reprs = [] - for element in iterator: - arg_repr = _log(names, element) + # problem: simple_response can also yield + if simple_threaded: + # remove self + names = names[1:] + def _response_simple_threaded(self, *args): + arg_repr = _log(names, args) if arg_repr: - arg_reprs.append(arg_repr) - if arg_reprs: - MODULE.info('%s got: [%s]' % (name, '], ['.join(', '.join(arg_repr) for arg_repr in arg_reprs))) - result = [] - for res in function(self, iterator, *args): - result.append(res) + logger('%s got: %s' % (name, ', '.join(arg_repr))) + result = function(self, *args) + res = result.next() + logger('%s returned: %r' % (name, res)) yield res - MODULE.info('%s returned: %r' % (name, result)) + for res in result: + yield res + _response = _response_simple_threaded + else: + # remove self, iterator + names = names[2:] + def _response_multiple(self, iterator, *args): + arg_reprs = [] + for element in iterator: + arg_repr = _log(names, element) + if arg_repr: + arg_reprs.append(arg_repr) + if arg_reprs: + logger('%s got: [%s]' % (name, '], ['.join(', '.join(arg_repr) for arg_repr in arg_reprs))) + result = [] + for res in function(self, iterator, *args): + result.append(res) + yield res + logger('%s returned: %r' % (name, result)) + _response = _response_multiple else: # remove self names = names[1:] - def _response(self, *args): + def _response_simple(self, *args): arg_repr = _log(names, args) if arg_repr: - MODULE.info('%s got: %s' % (name, ', '.join(arg_repr))) + logger('%s got: %s' % (name, ', '.join(arg_repr))) result = function(self, *args) - MODULE.info('%s returned: %r' % (name, result)) + logger('%s returned: %r' % (name, result)) return result + _response = _response_simple copy_function_meta_data(function, _response, copy_arg_inspect=True) return _response Index: umc/python/appcenter/__init__.py =================================================================== --- umc/python/appcenter/__init__.py (Revision 38793) +++ umc/python/appcenter/__init__.py (Arbeitskopie) @@ -38,14 +38,10 @@ import urllib2 import re -# related third party -import notifier -import notifier.threads - # univention from univention.lib.package_manager import PackageManager, LockError from univention.management.console.log import MODULE -from univention.management.console.modules.decorators import simple_response, sanitize, sanitize_list, multi_response +from univention.management.console.modules.decorators import simple_response, sanitize, sanitize_list, multi_response, log from univention.management.console.modules.sanitizers import PatternSanitizer, MappingSanitizer, DictSanitizer, StringSanitizer, ChoicesSanitizer, ListSanitizer, EmailSanitizer, BooleanSanitizer from univention.updater import UniventionUpdater from univention.updater.errors import ConfigurationError @@ -150,11 +146,11 @@ application=StringSanitizer(minimum=1, required=True), force=BooleanSanitizer() ) - def invoke(self, request): - function = request.options.get('function') - application_id = request.options.get('application') + @simple_response + @log(logger=MODULE.process, simple_threaded=True) + def invoke(self, function, application, force): + application_id = application application = Application.find(application_id) - force = request.options.get('force') try: # make sure that the application cane be installed/updated can_continue = True @@ -184,23 +180,18 @@ result['remove'] = application.uninstall_dry_run(self.package_manager) can_continue = False result['can_continue'] = can_continue - self.finished(request.id, result) + yield result + if can_continue: - def _thread(module, application, function): - with module.package_manager.locked(reset_status=True, set_finished=True): - with module.package_manager.no_umc_restart(exclude_apache=True): - if function in ('install', 'update'): - # dont have to add component: already added during dry_run - return application.install(module.package_manager, module.component_manager, add_component=False, send_as=function) - else: - return application.uninstall(module.package_manager, module.component_manager) - def _finished(thread, result): - if isinstance(result, BaseException): - MODULE.warn('Exception during %s %s: %s' % (function, application_id, str(result))) - thread = notifier.threads.Simple('invoke', - notifier.Callback(_thread, self, application, function), _finished) - thread.run() + with self.package_manager.locked(reset_status=True, set_finished=True): + with self.package_manager.no_umc_restart(exclude_apache=True): + if function in ('install', 'update'): + # dont have to add component: already added during dry_run + application.install(self.package_manager, self.component_manager, add_component=False, send_as=function) + else: + application.uninstall(self.package_manager, self.component_manager) + except LockError: # make it thread safe: another process started a package manager # this module instance already has a running package manager @@ -282,30 +273,23 @@ }, required=True), packages=ListSanitizer(StringSanitizer(minimum=1), required=True) ) - def packages_invoke(self, request): + @simple_response + @log(logger=MODULE.process, simple_threaded=True) + def packages_invoke(self, packages, function): """ executes an installer action """ - packages = request.options.get('packages') - function = request.options.get('function') try: with self.package_manager.locked(reset_status=True): not_found = [pkg_name for pkg_name in packages if self.package_manager.get_package(pkg_name) is None] - self.finished(request.id, {'not_found' : not_found}) + yield {'not_found' : not_found} if not not_found: - def _thread(package_manager, function, packages): - with package_manager.locked(set_finished=True): - with package_manager.no_umc_restart(): - if function == 'install': - package_manager.install(*packages) - else: - package_manager.uninstall(*packages) - def _finished(thread, result): - if isinstance(result, BaseException): - MODULE.warn('Exception during %s %s: %r' % (function, packages, str(result))) - thread = notifier.threads.Simple('invoke', - notifier.Callback(_thread, self.package_manager, function, packages), _finished) - thread.run() + with self.package_manager.locked(set_finished=True): + with self.package_manager.no_umc_restart(): + if function == 'install': + self.package_manager.install(*packages) + else: + self.package_manager.uninstall(*packages) except LockError: # make it thread safe: another process started a package manager # this module instance already has a running package manager