Index: univention-directory-manager-modules/modules/univention/admin/filter.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/filter.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/filter.py (Arbeitskopie) @@ -82,6 +82,12 @@ ''' return '%s(%r, %r)' % (self.__class__._type_, self.type, self.expressions) + def append_unmapped_filter_string(self, filter_s, rewrite_function, mapping): + if filter_s: + filter_p = parse(filter_s) + walk(filter_p, rewrite_function, arg=mapping) + self.expressions.append(filter_p) + class expression: """LDAP filter expression.""" _type_='expression' Index: univention-directory-manager-modules/modules/univention/admin/syntax.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/syntax.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/syntax.py (Arbeitskopie) @@ -274,6 +274,7 @@ depends = None error_message = _( "Not a valid LDAP DN" ) simple = False # by default a MultiObjectSelect widget is used; if simple == True a ComboBox is used + use_objects = True @classmethod def parse( self, text ): @@ -1603,9 +1604,11 @@ class GroupDN( UDM_Objects ): udm_modules = ( 'groups/group', ) + use_objects = False class UserDN( UDM_Objects ): udm_modules = ( 'users/user', ) + use_objects = False class HostDN( UDM_Objects ): udm_modules = ( 'computers/computer', ) @@ -1689,12 +1692,15 @@ udm_modules = ( 'dns/forward_zone', ) empty_value = True size = 'TwoThirds' + use_objects = False class DNS_ReverseZone( UDM_Objects ): description=_('DNS reverse zone') udm_modules = ( 'dns/reverse_zone', ) + label = '%(subnet)s' empty_value = True size = 'TwoThirds' + use_objects = False class dnsEntry( complex ): description=_('DNS Entry') @@ -1736,8 +1742,10 @@ class WritableShare( UDM_Objects ): udm_modules = ( 'shares/share', ) udm_filter = 'writeable=1' + label = _('%(name)s (%(path)s on %(host)s)') # ldap-optimized for shares/share.description() size = 'OneAndAHalf' empty_value = True + use_objects = False # class share(ldapDnOrNone): # searchFilter='(objectClass=univentionShare)' @@ -2550,7 +2558,6 @@ class nfsShare(UDM_Objects): udm_modules = ( 'shares/share', ) - key = 'dn' label = '%(printablename)s' udm_filter = 'objectClass=univentionShareNFS' Index: univention-directory-manager-modules/modules/univention/admin/handlers/groups/group.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/handlers/groups/group.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/handlers/groups/group.py (Arbeitskopie) @@ -940,23 +940,23 @@ self.lo.modify(dn, [ ('sambaPrimaryGroupSID', attr.get('sambaPrimaryGroupSID', []), [newSid]) ] ) self.update_sambaPrimaryGroupSid = False -def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): - - filter=univention.admin.filter.conjunction('&', [ - univention.admin.filter.expression('cn', '*'), - univention.admin.filter.conjunction('|', - [univention.admin.filter.conjunction('&', - [univention.admin.filter.expression('objectClass', 'univentionGroup'),]), - univention.admin.filter.conjunction('&', - [univention.admin.filter.expression('objectClass', 'sambaGroupMapping'),]) - ]) +def lookup_filter(filter_s=None): + lookup_filter = \ + univention.admin.filter.conjunction('&', [ + univention.admin.filter.expression('cn', '*'), + univention.admin.filter.conjunction('|', [ + univention.admin.filter.conjunction('&', + [univention.admin.filter.expression('objectClass', 'univentionGroup'),]), + univention.admin.filter.conjunction('&', + [univention.admin.filter.expression('objectClass', 'sambaGroupMapping'),]) + ]) ]) + lookup_filter.append_unmapped_filter_string(filter_s, univention.admin.mapping.mapRewrite, mapping) + return lookup_filter - if filter_s: - filter_p=univention.admin.filter.parse(filter_s) - univention.admin.filter.walk(filter_p, univention.admin.mapping.mapRewrite, arg=mapping) - filter.expressions.append(filter_p) +def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): + filter=lookup_filter(filter_s) res=[] for dn, attrs in lo.search(unicode(filter), base, scope, [], unique, required, timeout, sizelimit): res.append( object( co, lo, None, dn, attributes = attrs ) ) Index: univention-directory-manager-modules/modules/univention/admin/handlers/dns/forward_zone.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/handlers/dns/forward_zone.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/handlers/dns/forward_zone.py (Arbeitskopie) @@ -326,20 +326,20 @@ if not self.hasChanged('serial'): self['serial']=str(int(self['serial'])+1) -def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): - - filter=univention.admin.filter.conjunction('&', [ - univention.admin.filter.expression('objectClass', 'dNSZone'), - univention.admin.filter.expression('relativeDomainName', '@'), - univention.admin.filter.conjunction('!', [univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP4)]), - univention.admin.filter.conjunction('!', [univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP6)]), +def lookup_filter(filter_s=None): + lookup_filter = \ + univention.admin.filter.conjunction('&', [ + univention.admin.filter.expression('objectClass', 'dNSZone'), + univention.admin.filter.expression('relativeDomainName', '@'), + univention.admin.filter.conjunction('!', [univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP4)]), + univention.admin.filter.conjunction('!', [univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP6)]), ]) + lookup_filter.append_unmapped_filter_string(filter_s, univention.admin.mapping.mapRewrite, mapping) + return lookup_filter - if filter_s: - filter_p=univention.admin.filter.parse(filter_s) - univention.admin.filter.walk(filter_p, univention.admin.mapping.mapRewrite, arg=mapping) - filter.expressions.append(filter_p) +def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): + filter=lookup_filter(filter_s) res=[] for dn, attrs in lo.search(unicode(filter), base, scope, [], unique, required, timeout, sizelimit): res.append((object(co, lo, None, dn=dn, superordinate=superordinate, attributes = attrs ))) Index: univention-directory-manager-modules/modules/univention/admin/handlers/dns/reverse_zone.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/handlers/dns/reverse_zone.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/handlers/dns/reverse_zone.py (Arbeitskopie) @@ -286,22 +286,22 @@ return unmapSubnet(rdn_value) -def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): - - filter=univention.admin.filter.conjunction('&', [ - univention.admin.filter.expression('objectClass', 'dNSZone'), - univention.admin.filter.expression('relativeDomainName', '@'), - univention.admin.filter.conjunction('|', [ - univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP4), - univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP6) +def lookup_filter(filter_s=None): + lookup_filter = \ + univention.admin.filter.conjunction('&', [ + univention.admin.filter.expression('objectClass', 'dNSZone'), + univention.admin.filter.expression('relativeDomainName', '@'), + univention.admin.filter.conjunction('|', [ + univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP4), + univention.admin.filter.expression('zoneName', '*%s' % ARPA_IP6) ]), ]) + lookup_filter.append_unmapped_filter_string(filter_s, univention.admin.mapping.mapRewrite, mapping) + return lookup_filter - if filter_s: - filter_p=univention.admin.filter.parse(filter_s) - univention.admin.filter.walk(filter_p, univention.admin.mapping.mapRewrite, arg=mapping) - filter.expressions.append(filter_p) +def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): + filter=lookup_filter(filter_s) res=[] for dn, attrs in lo.search(unicode(filter), base, scope, [], unique, required, timeout, sizelimit): res.append((object(co, lo, None, dn=dn, superordinate=superordinate, attributes = attrs ))) Index: univention-directory-manager-modules/modules/univention/admin/handlers/shares/share.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/handlers/shares/share.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/handlers/shares/share.py (Arbeitskopie) @@ -900,7 +900,6 @@ if not hasattr(self,"options"): self.open() if 'nfs' in self.options: - ulist=[] searchstring="*"+self['host']+":"+self['path']+"*" searchResult=self.lo.searchDn(base=self.position.getDomain(), filter='(&(objectClass=person)(automountInformation=%s))'%searchstring, scope='domain') if searchResult: @@ -932,17 +931,17 @@ def description(self): return _('%s (%s on %s)') % (self['name'], self['path'], self['host']) -def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): - filter=univention.admin.filter.conjunction('&', [ - univention.admin.filter.expression('objectClass', 'univentionShare'), - univention.admin.filter.expression('cn', '*'), +def lookup_filter(filter_s=None): + lookup_filter = \ + univention.admin.filter.conjunction('&', [ + univention.admin.filter.expression('objectClass', 'univentionShare'), + univention.admin.filter.expression('cn', '*'), ]) + lookup_filter.append_unmapped_filter_string(filter_s, univention.admin.mapping.mapRewrite, mapping) + return lookup_filter - if filter_s: - filter_p=univention.admin.filter.parse(filter_s) - univention.admin.filter.walk(filter_p, univention.admin.mapping.mapRewrite, arg=mapping) - filter.expressions.append(filter_p) - +def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): + filter=lookup_filter(filter_s) res=[] for dn in lo.searchDn(unicode(filter), base, scope, unique, required, timeout, sizelimit): res.append(object(co, lo, None, dn)) Index: univention-directory-manager-modules/modules/univention/admin/handlers/users/user.py =================================================================== --- univention-directory-manager-modules/modules/univention/admin/handlers/users/user.py (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/handlers/users/user.py (Arbeitskopie) @@ -2759,32 +2759,32 @@ else: univention.admin.mapping.mapRewrite(filter, mapping) -def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): - - filter=univention.admin.filter.conjunction('&', [ - univention.admin.filter.conjunction('|', [ - univention.admin.filter.conjunction('&', [ - univention.admin.filter.expression('objectClass', 'posixAccount'), - univention.admin.filter.expression('objectClass', 'shadowAccount'), +def lookup_filter(filter_s=None): + lookup_filter = \ + univention.admin.filter.conjunction('&', [ + univention.admin.filter.conjunction('|', [ + univention.admin.filter.conjunction('&', [ + univention.admin.filter.expression('objectClass', 'posixAccount'), + univention.admin.filter.expression('objectClass', 'shadowAccount'), + ]), + univention.admin.filter.expression('objectClass', 'univentionMail'), + univention.admin.filter.expression('objectClass', 'sambaSamAccount'), + univention.admin.filter.expression('objectClass', 'simpleSecurityObject'), + univention.admin.filter.conjunction('&', [ + univention.admin.filter.expression('objectClass', 'person'), + univention.admin.filter.expression('objectClass', 'organizationalPerson'), + univention.admin.filter.expression('objectClass', 'inetOrgPerson'), + ]), ]), - univention.admin.filter.expression('objectClass', 'univentionMail'), - univention.admin.filter.expression('objectClass', 'sambaSamAccount'), - univention.admin.filter.expression('objectClass', 'simpleSecurityObject'), - univention.admin.filter.conjunction('&', [ - univention.admin.filter.expression('objectClass', 'person'), - univention.admin.filter.expression('objectClass', 'organizationalPerson'), - univention.admin.filter.expression('objectClass', 'inetOrgPerson'), - ]), - ]), - univention.admin.filter.conjunction('!', [univention.admin.filter.expression('uidNumber', '0')]), - univention.admin.filter.conjunction('!', [univention.admin.filter.expression('uid', '*$')]), - ]) + univention.admin.filter.conjunction('!', [univention.admin.filter.expression('uidNumber', '0')]), + univention.admin.filter.conjunction('!', [univention.admin.filter.expression('uid', '*$')]), + ]) + # ATTENTION: has its own rewrite function. + lookup_filter.append_unmapped_filter_string(filter_s, rewrite, mapping) + return lookup_filter - if filter_s: - filter_p=univention.admin.filter.parse(filter_s) - univention.admin.filter.walk(filter_p, rewrite, arg=mapping) - filter.expressions.append(filter_p) - +def lookup(co, lo, filter_s, base='', superordinate=None, scope='sub', unique=0, required=0, timeout=-1, sizelimit=0): + filter=lookup_filter(filter_s) res=[] for dn, attrs in lo.search(unicode(filter), base, scope, [], unique, required, timeout, sizelimit): res.append( object( co, lo, None, dn, attributes = attrs ) ) Index: univention-directory-manager-modules/modules/univention/admin/de.po =================================================================== --- univention-directory-manager-modules/modules/univention/admin/de.po (Revision 40501) +++ univention-directory-manager-modules/modules/univention/admin/de.po (Arbeitskopie) @@ -221,6 +221,10 @@ msgid "cannot find fqdn of " msgstr "FQDN konnte nicht gefunden werden: " +#: syntax.py:1743 +msgid "%(name)s (%(path)s on %(host)s)" +msgstr "%(name)s (%(path)s auf %(host)s)" + #: syntax.py:109 syntax.py:162 syntax.py:298 msgid "Invalid value" msgstr "Ungültiger Wert" Index: univention-directory-manager-modules/debian/changelog =================================================================== --- univention-directory-manager-modules/debian/changelog (Revision 40501) +++ univention-directory-manager-modules/debian/changelog (Arbeitskopie) @@ -1,3 +1,9 @@ +univention-directory-manager-modules (8.0.155-1) unstable; urgency=low + + * Bug #30991: Preliminary work for agressive syntax lookup optimizations + + -- Dirk Wiesenthal Tue, 14 May 2013 19:27:56 +0200 + univention-directory-manager-modules (8.0.154-1) unstable; urgency=low * Bug #31154: Import hook files every time udm_modules.update() is called Index: univention-management-console-module-udm/debian/changelog =================================================================== --- univention-management-console-module-udm/debian/changelog (Revision 40501) +++ univention-management-console-module-udm/debian/changelog (Arbeitskopie) @@ -1,3 +1,9 @@ +univention-management-console-module-udm (3.0.79-1) unstable; urgency=low + + * Bug #30991: Greatly speedup syntax lookup if syntax allows it + + -- Dirk Wiesenthal Tue, 14 May 2013 19:37:46 +0200 + univention-management-console-module-udm (3.0.78-1) unstable; urgency=low * Bug #14123: adjusted texts for GPL license Index: univention-management-console-module-udm/umc/python/udm/udm_ldap.py =================================================================== --- univention-management-console-module-udm/umc/python/udm/udm_ldap.py (Revision 40501) +++ univention-management-console-module-udm/umc/python/udm/udm_ldap.py (Arbeitskopie) @@ -232,6 +232,12 @@ module = self._initialized_with_module self.module = _module_cache.get( module, force_reload=force_reload ) + def allows_simple_lookup( self ): + return self.lookup_filter() is not None + + def lookup_filter( self, filter_s=None ): + return getattr( self.module, 'lookup_filter', lambda x: None )(filter_s) + def __getitem__( self, key ): props = getattr( self.module, 'property_descriptions', {} ) return props[ key ] @@ -407,7 +413,7 @@ raise UDM_Error( get_exception_msg(e) ) @LDAP_Connection - def search( self, container = None, attribute = None, value = None, superordinate = None, scope = 'sub', filter = '', ldap_connection = None, ldap_position = None ): + def search( self, container = None, attribute = None, value = None, superordinate = None, scope = 'sub', filter = '', simple = False, simple_attrs = None, ldap_connection = None, ldap_position = None ): """Searches for LDAP objects based on a search pattern""" if container == 'all': container = ldap_position.getBase() @@ -421,7 +427,14 @@ result = None try: sizelimit = int(ucr.get('directory/manager/web/sizelimit', '2000')) - result = self.module.lookup( None, ldap_connection, filter_s, base = container, superordinate = superordinate, scope = scope, sizelimit = sizelimit ) + if simple and self.allows_simple_lookup(): + lookup_filter = unicode(self.lookup_filter(filter)) + if simple_attrs is not None: + result = ldap_connection.search( filter = lookup_filter, base = container, scope = scope, sizelimit = sizelimit, attr = simple_attrs ) + else: + result = ldap_connection.searchDn( filter = lookup_filter, base = container, scope = scope, sizelimit = sizelimit ) + else: + result = self.module.lookup( None, ldap_connection, filter_s, base = container, superordinate = superordinate, scope = scope, sizelimit = sizelimit ) except udm_errors.insufficientInformation, e: return [] except udm_errors.ldapTimeout, e: @@ -1004,7 +1017,7 @@ filter_s = '(&%s%s)' % (property_filter_s, filter_s) return filter_s -LDAP_ATTR_RE = re.compile(r'^%\(([^(]*)\)s$') # '%(username)s' -> 'username' +LDAP_ATTR_RE = re.compile(r'^%\(([^)]*)\)s$') # '%(username)s' -> 'username' def _get_syntax( syntax_name ): if syntax_name not in udm_syntax.__dict__: return None @@ -1051,7 +1064,7 @@ filter_s = _create_ldap_filter( syn, options, module ) if filter_s is not None: try: - size += len( module.search( filter = filter_s ) ) + size += len( module.search( filter = filter_s, simple=not syn.use_objects ) ) except udm_errors.ldapSizelimitExceeded: return {'performs_well' : True, 'size_limit_exceeded' : True} return {'size' : size, 'performs_well' : True } @@ -1065,46 +1078,104 @@ if issubclass( syn.__class__, udm_syntax.UDM_Objects ): syn.choices = [] - def map_choices( obj_list ): - result = [] - for obj in obj_list: - obj.open() + # try to avoid using the slow udm interface + simple = False + attr = set() + if not syn.use_objects: + attr.update(re.findall(r'%\(([^)]+)\)', syn.key)) + if syn.label: + attr.update(re.findall(r'%\(([^)]+)\)', syn.label)) + for udm_module in syn.udm_modules: + module = UDM_Module( udm_module ) + if not module.allows_simple_lookup(): + break + if module is not None: + mapping = module.module.mapping + if not all([mapping.mapName(att) for att in attr]): + break + else: + simple = True + def extract_key_label(syn, dn, info): + key = label = None + if syn.key == 'dn': + key = dn + else: + try: + key = syn.key % info + except KeyError: + pass + if syn.label == 'dn': + label = dn + elif syn.label is None: + pass + else: + try: + label = syn.label % info + except KeyError: + pass + return key, label + if not simple: + def map_choices( obj_list ): + result = [] + for obj in obj_list: + # first try it without obj.open() (expensive) + key, label = extract_key_label(syn, obj.dn, obj.info) + if key is None or label is None: + obj.open() + key, label = extract_key_label(syn, obj.dn, obj.info) + if key is None: + # ignore the entry as the key is important for a selection, there + # is no sensible fallback for the key (Bug #26994) + continue + if label is None: + # fallback to the default description as this is just what displayed + # to the user (Bug #26994) + label = udm_objects.description( obj ) + result.append( (key, label) ) + return result - if syn.key == 'dn': - key = obj.dn + for udm_module in syn.udm_modules: + module = UDM_Module( udm_module ) + if module is None: + continue + filter_s = _create_ldap_filter( syn, options, module ) + if filter_s is None: + syn.choices = [] else: - try: - key = syn.key % obj.info - except KeyError: - # ignore the entry as the key is important for a selection, there - # is no sensible fallback for the key (Bug #26994) - continue - if syn.label is None: - label = udm_objects.description( obj ) - elif syn.label == 'dn': - label = obj.dn - else: - try: - label = syn.label % obj.info - except KeyError: - # fallback to the default description as this is just what displayed - # to the user (Bug #26994) - label = udm_objects.description( obj ) - - result.append( (key, label) ) - return result - - for udm_module in syn.udm_modules: - module = UDM_Module( udm_module ) - if module is None: - continue - filter_s = _create_ldap_filter( syn, options, module ) - if filter_s is None: - syn.choices = [] - else: - search_options = {'filter' : filter_s} - search_options.update(module_search_options) - syn.choices.extend( map_choices( module.search( **search_options ) ) ) + search_options = {'filter' : filter_s} + search_options.update(module_search_options) + syn.choices.extend( map_choices( module.search( **search_options ) ) ) + else: + for udm_module in syn.udm_modules: + module = UDM_Module( udm_module ) + if module is None: + continue + filter_s = _create_ldap_filter( syn, options, module ) + if filter_s is not None: + if filter_s and not filter_s.startswith('('): + filter_s = '(%s)' % filter_s + mapping = module.module.mapping + ldap_attr = [mapping.mapName(att) for att in attr] + search_options = {'filter' : filter_s, 'simple' : True} + search_options.update(module_search_options) + if ldap_attr: + search_options['simple_attrs'] = ldap_attr + result = module.search( **search_options ) + for dn, ldap_map in result: + info = univention.admin.mapping.mapDict(mapping, ldap_map) + key, label = extract_key_label(syn, dn, info) + if key is None: + continue + if label is None: + label = ldap_connection.explodeDn(dn, 1)[0] + syn.choices.append((key, label)) + else: + keys = module.search( **search_options ) + if syn.label == 'dn': + labels = keys + else: + labels = [ldap_connection.explodeDn(dn, 1)[0] for dn in keys] + syn.choices.extend(zip(keys, labels)) if isinstance( syn.static_values, ( tuple, list ) ): for value in syn.static_values: syn.choices.insert( 0, value )