Index: python/univention-updater =================================================================== --- python/univention-updater (Revision 14758) +++ python/univention-updater (Arbeitskopie) @@ -129,41 +129,19 @@ if os.path.exists(os.path.join(tempdir,sh)): os.remove(os.path.join(tempdir,sh)) - proxy_headers = updater.open_connection() - path="%s/%s" % ( server_path, sh ) - site = '%s/%s/%s' % (updater.proxy_prefix, updater.repository_prefix, path) - replace_slash = re.compile ('[/]{2,}') - site = replace_slash.sub ('/', site) - if not site.startswith ('http://') and proxy_headers != None: - site = 'http://%s' % site - if proxy_headers != None: - updater.connection.putrequest('GET', site, skip_accept_encoding=1) - else: - updater.connection.putrequest('GET', site) - - if proxy_headers != None: - for k, v in proxy_headers.items (): - updater.connection.putheader (k, v) - try: - updater.connection.endheaders () - response = updater.connection.getresponse() - response_data = response.read() - - if response.status != 200: - dprint('failed to download %s' % path) - dprint(' %s: %s' % (response.status,response.reason)) - continue + (response, data) = updater.repository().access(server_path, sh, method="GET") + if response.status != 200: + dprint('failed to download %s' % path) + dprint(' %s: %s' % (response.status,response.reason)) + continue except: dprint( traceback.format_exc ()) f=open(os.path.join(tempdir, sh), 'w+' ) - f.write(response_data) + f.write(data) f.close() os.chmod(os.path.join(tempdir, sh), 0744) - updater.close_connection() - - def update_available(mode, baseConfig, cdrom_mount_point, sourcedir, stdout, sys_stdout, reboot, internal=1, iso=None, updater=None): # Checks if there is an update available @@ -284,9 +262,9 @@ code, message = msg if code == socket.EAI_NONAME: - dprint('Error: The name %s is not known.\n Please check your network settings or set the UCR variable repository/online/server to a diffrent value.' % (updater.repository_server), [ sys.stdout, stdout ]) + dprint('Error: The name %s is not known.\n Please check your network settings or set the UCR variable repository/online/server to a diffrent value.' % (updater.repository()), [ sys.stdout, stdout ]) else: - dprint('Error: An error occurs during the connection to %s.\n Please check your network settings and check the logfile /var/log/univention/updater.log' % (updater.repository_server), [ sys.stdout, stdout ]) + dprint('Error: An error occurs during the connection to %s.\n Please check your network settings and check the logfile /var/log/univention/updater.log' % (updater.repository()), [ sys.stdout, stdout ]) sys.exit( 1 ) @@ -516,14 +494,11 @@ for maintained in maintained_state_list: for arch in ['all', architecture, 'extern' ]: if updater.net_path_exists('%s/%s/%s-%s/%s/' % (next_vv, maintained, next_vv, next_vp, arch)): - if updater.repository_prefix: - add_temporary_sources_list ('deb http://%s/%s/%s/%s/ %s/%s/' % ( updater.repository_server, updater.repository_prefix, next_vv, maintained, nextversion, arch)) - else: - add_temporary_sources_list ('deb http://%s/%s/%s/ %s/%s/' % ( updater.repository_server, next_vv, maintained, nextversion, arch)) + add_temporary_sources_list ('deb %s/%s/%s/ %s/%s/' % ( updater.repository(), next_vv, maintained, nextversion, arch)) # Download the pre and postup scripts scriptPath = tempdir - download_sh_files(updater, '%s/%s/maintained/%s-%s/all' % (updater.repository_prefix, next_vv, next_vv, next_vp), tempdir) + download_sh_files(updater, '%s/maintained/%s-%s/all' % (next_vv, next_vv, next_vp), tempdir) dprint('**** Starting actual update at %s' % datetime.datetime.now ().ctime ()) Index: python/univention-repository-migrate =================================================================== --- python/univention-repository-migrate (Revision 14754) +++ python/univention-repository-migrate (Arbeitskopie) @@ -31,16 +31,13 @@ from optparse import OptionParser import copy import os -import re import shutil -import string -import subprocess import sys import time import univention.config_registry as ucr import univention.updater.repository as urepo -from univention.updater import UniventionMirror, UCS_Version +from univention.updater import UCS_Version configRegistry = ucr.ConfigRegistry() configRegistry.load() Index: python/univention-security-update =================================================================== --- python/univention-security-update (Revision 14754) +++ python/univention-security-update (Arbeitskopie) @@ -194,9 +194,9 @@ code, message = msg if code == socket.EAI_NONAME: - dprint( 'Error: The name %s is not known.\n Please check your network settings or set the UCR variable repository/online/server to a diffrent value.' % (updater.repository_server), [fp_debug,sys.stderr] ) + dprint( 'Error: The name %s is not known.\n Please check your network settings or set the UCR variable repository/online/server to a diffrent value.' % (updater.repository()), [fp_debug,sys.stderr] ) else: - dprint( 'Error: An error occurs during the connection to %s.\n Please check your network settings and check the logfile /var/log/univention/security-updates.log' % (updater.repository_server), [fp_debug,sys.stderr] ) + dprint( 'Error: An error occurs during the connection to %s.\n Please check your network settings and check the logfile /var/log/univention/security-updates.log' % (updater.repository()), [fp_debug,sys.stderr] ) sys.exit( 1 ) Index: python/univention-repository-merge =================================================================== --- python/univention-repository-merge (Revision 14754) +++ python/univention-repository-merge (Arbeitskopie) @@ -30,11 +30,9 @@ import os, string import shutil, sys -import getopt, time +import getopt -import univention.config_registry as ucr import univention.updater.repository as urepo -from univention.updater import UniventionMirror, UCS_Version l_package={} p_remove=[] @@ -174,4 +172,3 @@ os.chdir(packages_dir) urepo.update_indexes( packages_dir, stdout = fp_debug, stderr = fp_debug ) fp_debug.close() - Index: debian/univention-updater.univention-config-registry =================================================================== --- debian/univention-updater.univention-config-registry (Revision 14992) +++ debian/univention-updater.univention-config-registry (Arbeitskopie) @@ -57,3 +57,4 @@ Variables: repository/mirror/threads Variables: repository/mirror/architectures Variables: repository/mirror/version/.* +Variabled: online/repository/clean Index: conffiles/etc/apt/mirror.list =================================================================== --- conffiles/etc/apt/mirror.list (Revision 14992) +++ conffiles/etc/apt/mirror.list (Arbeitskopie) @@ -16,7 +16,7 @@ try: mirror = UniventionMirror() - if mirror.online_repository in ['false', '0', 'no', 'disabled']: + if not mirror.online_repository: print '# The mirror mode is disabled and can be enabled with:' print '# univention-config-registry set repository/mirror=true' print '' Index: modules/univention/updater/mirror.py =================================================================== --- modules/univention/updater/mirror.py (Revision 14758) +++ modules/univention/updater/mirror.py (Arbeitskopie) @@ -35,60 +35,35 @@ from tools import UniventionUpdater, UCS_Version class UniventionMirror( UniventionUpdater ): + """ Handle /etc/apt/mirror.list """ def __init__( self ): UniventionUpdater.__init__( self ) - self.online_repository = self.configRegistry.get( 'repository/mirror', 'yes' ) - self.repository_server = self.configRegistry.get( 'repository/mirror/server', 'apt.univention.de' ) self.repository_path = self.configRegistry.get( 'repository/mirror/basepath', '/var/lib/univention-repository' ) - self.repository_prefix = self.configRegistry.get( 'repository/mirror/prefix', '' ) - if not self.repository_prefix and self.net_path_exists( '/univention-repository/' ): - self.repository_prefix = 'univention-repository' - if self.configRegistry.has_key( 'repository/mirror/version/end' ): - self.version_end = UCS_Version( self.configRegistry.get( 'repository/mirror/version/end' ) ) - else: - self.version_end = UCS_Version( ( self.version_major, self.version_minor, self.patchlevel ) ) - if self.configRegistry.has_key( 'repository/mirror/version/start' ): - self.version_start = UCS_Version( self.configRegistry.get( 'repository/mirror/version/start' ) ) - else: - self.version_start = UCS_Version( ( self.version_major, 0, 0 ) ) + self.version_end = UCS_Version( + self.configRegistry.get('repository/mirror/version/end', + (self.version_major, self.version_minor, self.patchlevel) ) + ) + self.version_start = UCS_Version( + self.configRegistry.get('repository/mirror/version/start', + (self.version_major, 0, 0) ) ) # set architectures to mirror - archs = self.configRegistry.get( 'repository/mirror/architectures', '' ) - if archs: - self.architectures = archs.split( ' ' ) + archs = self.configRegistry.get('repository/mirror/architectures', '').split(' ') + + def config_repository( self ): + """ Retrieve configuration to access repository. Overrides UniventionUpdater. """ + self.online_repository = self.ucr_bool('repository/mirror', True) + self.repositoryFromRegistry(self.configRegistry, 'repository/online') def retrieve_url( self, path ): '''downloads the given path from the repository server''' - # path MUST NOT contain the schema and hostname - proxy_headers = self.open_connection() - site = '%s/%s/%s' % (self.proxy_prefix, self.repository_prefix, path) - - replace_slash = re.compile ('[/]{2,}') - site = replace_slash.sub ('/', site) - if not site.startswith ('http://') and proxy_headers != None: - site = 'http://%s' % site - - if proxy_headers != None: - self.connection.putrequest('GET', site, skip_accept_encoding=1) - else: - self.connection.putrequest('GET', site) - - if proxy_headers != None: - for k, v in proxy_headers.items (): - self.connection.putheader (k, v) try: - self.connection.endheaders () - response = self.connection.getresponse() - body = response.read() - + (response, data) = self.repository().access(path, method='GET') if response.status == 200: - self.close_connection() - return body + return data except: import traceback print traceback.format_exc () - - self.close_connection() return None def copy_script( self, script, repository, directory ): @@ -218,3 +193,7 @@ self.mirror_repositories() self.mirror_update_scripts() self.update_dists_files() +if __name__ == '__main__': + umi = UniventionMirror() + import doctest + doctest.testmod() Index: modules/univention/updater/tools.py =================================================================== --- modules/univention/updater/tools.py (Revision 14763) +++ modules/univention/updater/tools.py (Arbeitskopie) @@ -47,8 +47,14 @@ def __init__( self, version ): '''version must a string matching the pattern X.Y-Z or a triple - with major, minor and patchlevel''' + with major, minor and patchlevel + >>> str(UCS_Version('2.3-0')) + '2.3-0' + >>> str(UCS_Version((2,3,0))) + '2.3-0' + ''' + if isinstance( version, ( tuple, list ) ) and len( version ) == 3: self.major = int( version[ 0 ] ) self.minor = int( version[ 1 ] ) @@ -59,7 +65,13 @@ def __cmp__( self, right ): '''Compare to UCS versions. The method returns 0 if the versions are equal, -1 if the left is less than the right and 1 of the - left is greater than the right''' + left is greater than the right + + >>> UCS_Version('2.2-0') < UCS_Version('2.2-1') + True + >>> UCS_Version('2.2-0') > UCS_Version('2.3-0') + False + ''' # major version differ if self.major < right.major: return -1 @@ -79,6 +91,7 @@ return 0 def set( self, version ): + """Convert "version" string into its parts.""" match = UCS_Version._regexp.match( version ) if not match: raise AttributeError( 'string does not match UCS version pattern' ) @@ -91,172 +104,230 @@ def __str__( self ): return '%d.%d-%d' % ( self.major, self.minor, self.patchlevel ) -class UniventionUpdater: +class ResolvableUrl: + """ Represents an URL of a repository """ + failed = {} - def __init__(self): - self.connection = None - self.proxy_prefix = None - self.proxy_username = None - self.proxy_password = None - self.proxy_server = None - self.proxy_port = None + def __init__(self, baseurl, username=None, password=None, proxy=None): + """ Initialize APT source. + url: REQUIRED + username: None + password: None + proxy: OPTIONAL, e.g. [http://][user:password@]host[:port] + """ + self.baseurl = baseurl.rstrip('/') + '/' + self.username = username + self.password = password + if proxy != None and proxy.lower().startswith('http://'): + self.proxy_netloc = proxy[len('http://'):] + else: + self.proxy_netloc = proxy + if self.proxy_netloc: + if self.proxy_netloc.find ('@') != -1: + user_pwd, self.proxy_netloc = location.split ('@') + self.proxy_username, self.proxy_password = user_pwd.split(':') + else: + self.proxy_username, self.proxy_password = (None, None) - self.ucr_reinit() + def __str__( self ): + """Return a nice string representation of the url.""" + (scheme, netloc, path, query, fragment) = httplib.urlsplit(self.baseurl) + s = [ scheme, '://' ] + if self.username and self.password: + s += [ self.username, ':', self.password, '@' ] + s += [ netloc, path ] + if query: + s += ['?', query] + if fragment: + s += ['?', fragment] + return ''.join(s) - def open_connection(self, server=None, port=None): - '''Open http-connection to server:port''' + def access(self, *path, **options): + """ Access url and return 2-tuple (respone, response.data). + optional method overrides default HTTP-method HEAD """ + url = self.baseurl + '/'.join( map(lambda x:x.lstrip('/'), filter(None, path) ) ) + method = options.get('method','HEAD') + (scheme, netloc, path, query, fragment) = httplib.urlsplit(url) + # if we already know a net-location being bad, re-raise exception + if ResolvableUrl.failed.has_key(netloc): + raise ResolvableUrl.failed[netloc] + try: + try: + if self.proxy_netloc != None: + connection = httplib.HTTPConnection(self.proxy_netloc) + connection.putrequest(method, url, skip_accept_encoding=True) + if self.proxy_username != None and self.proxy_password != None: + user_pass = base64.encodestring('%s:%s' % (self.proxy_username, self.proxy_password)).rstrip() + connection.putheader('Proxy-Authorization', 'Basic %s' % user_pass) + else: + connection = httplib.HTTPConnection(netloc) + connection.putrequest(method, path) + if self.username != None and self.password != None: + user_pass = base64.encodestring('%s:%s' % (self.username, self.password)).rstrip() + connection.putheader('Authorization', 'Basic %s' % user_pass) + connection.endheaders() + response = connection.getresponse() + response_data = response.read() + return (response, response_data) + except socket.gaierror, e: + ResolvableUrl.failed[netloc] = e + raise e + except socket.error, e: + ResolvableUrl.failed[netloc] = e + raise e + finally: + connection.close() - if not self.nameserver_available: - raise socket.gaierror, (socket.EAI_NONAME, 'The repository server %s could not be resolved.' % server) - if not server: - server = self.repository_server - if port in (None, ''): - port = self.repository_port + def exists(self, *path): + """ Test if path exists on repository """ + m = {'method':'HEAD'} + (response, data) = self.access(*path, **m) + return response.status == httplib.OK - if self.proxy and self.proxy != '': - self.proxy_prefix = "%s:%s" % (server, port) +class RepositoryConfig: + """ Repository configuration """ + def __init__(self): + self.repository_server = None + self.repository_port = None + self.repository_prefix = None + self.repository_username = None + self.repository_password = None + self.repository_proxy = None + self._repository_hash = 0 + self._repository = None - location = self.proxy - if location.find ('@') != -1: - user_pwd, location = location.split ('@') - self.proxy_username, self.proxy_password = user_pwd.split(':') + def repository(self): + h = hash((self.repository_server, self.repository_port, + self.repository_prefix, self.repository_username, + self.repository_password, self.repository_proxy)) + if h != self._repository_hash: + self._repository_hash = h + self._repository = self._repositoryFromSelf() + return self._repository - if location.find (':') != -1: - location, pport = location.split (':') - self.proxy_port = int (pport) - else: - self.proxy_port = HTTP_PROXY_DEFAULT_PORT - self.proxy_server = location + def _repositoryFromSelf(self): + """ Create repository URL from self """ + if self.repository_prefix not in (None, "none", "/"): + url = 'http://%s:%s/%s/' % (self.repository_server, + self.repository_port, self.repository_prefix.strip('/')) + else: + url = 'http://%s:%s/' % (self.repository_server, self.repository_port) + return ResolvableUrl(url, username=self.repository_username, + password=self.repository_password, proxy=self.repository_proxy) - self.connection = httplib.HTTPConnection(self.proxy_server, self.proxy_port) - proxy_headers = {} + def repositoryFromRegistry(self, configRegistry, basepath, altPrefix='/univention-repository/'): + """ Create repository URL from configRegistry[basepath/*] """ + self.repository_server = configRegistry.get('%s/server' % basepath, 'apt.univention.de') + self.repository_port = configRegistry.get('%s/port' % basepath, '80') + self.repository_prefix = configRegistry.get('%s/prefix' % basepath, '') + self.repository_username = configRegistry.get('%s/username' % basepath, None) + self.repository_password = configRegistry.get('%s/password' % basepath, None) + # check for prefix on repository server (if the repository server is reachable) + if not self.repository_prefix: + u = self._repositoryFromSelf() + try: + if u.exists(altPrefix.strip('/')): + self.repository_prefix = altPrefix + except: + pass - if self.proxy_username and self.proxy_password: - #setup basic authentication - user_pass = base64.encodestring('%s:%s' % (self.proxy_username, self.proxy_password)) - proxy_headers['Proxy-Authorization'] = string.strip ('Basic %s' % user_pass) - return proxy_headers - else: - self.connection = httplib.HTTPConnection(server, int(port)) +class UniventionUpdater(RepositoryConfig): + """ Handle /etc/apt/sources.list.d/* """ + def __init__(self): + RepositoryConfig.__init__(self) + self.architectures = [ os.popen('dpkg-architecture -qDEB_BUILD_ARCH 2>/dev/null').readline()[:-1] ] + self.ucr_reinit() - def close_connection(self): - '''Close http-connection''' - self.connection.close() + def ucr_bool(self, key, default=False): + '''Retrieve boolean value from Univention Configuration Registry + >>> uup.ucr_bool('repository/online', True) + True + ''' + value = self.configRegistry.get(key, None) + if value == None: + return default + return value.lower() in ('true', 'yes', 'enable', 'enabled', '1') def ucr_reinit(self): '''Re-initialize settings''' - self.configRegistry=univention.config_registry.ConfigRegistry() + self.configRegistry = univention.config_registry.ConfigRegistry() self.configRegistry.load() - self.architectures = [ os.popen('dpkg-architecture -qDEB_BUILD_ARCH 2>/dev/null').readline()[:-1] ] - - self.online_repository=self.configRegistry.get('repository/online', 'True') - if self.online_repository.lower() in ['true', 'yes', '1']: - self.online_repository = True - else: - self.online_repository = False - - self.repository_server = self.configRegistry.get('repository/online/server', 'apt.univention.de') - self.repository_port = self.configRegistry.get('repository/online/port', '80') - self.repository_prefix = self.configRegistry.get('repository/online/prefix', '') - self.is_repository_server = self.configRegistry.get( 'local/repository', 'no' ) in ( 'yes', 'true' ) - if self.configRegistry.has_key('proxy/http') and self.configRegistry['proxy/http']: - self.proxy = self.configRegistry['proxy/http'].lower().replace('http://','') + self.repository_proxy = self.configRegistry['proxy/http']; elif os.environ.has_key('http_proxy') and os.environ['http_proxy']: - self.proxy = os.environ['http_proxy'].lower().replace('http://','') + self.repository_proxy = os.environ['http_proxy']; else: - self.proxy = None - self.proxy_prefix = '' + self.repository_proxy = None + self.is_repository_server = self.ucr_bool('local/repository', False) + # check for maintained and unmaintained self.parts = [] - maintained = self.configRegistry.get('repository/online/maintained', 'True') - if maintained.lower() in ['true', 'yes', '1']: + if self.ucr_bool('repository/online/maintained', True): self.parts.append('maintained') - unmaintained = self.configRegistry.get('repository/online/unmaintained', 'False') - if unmaintained.lower() in ['true', 'yes', '1']: + if self.ucr_bool('repository/online/unmaintained', False): self.parts.append('unmaintained') #UCS version - self.ucs_version=self.configRegistry['version/version'] - self.patchlevel=self.configRegistry['version/patchlevel'] - self.security_patchlevel=self.configRegistry['version/security-patchlevel'] - self.version_major = self.ucs_version.split('.')[0] - self.version_minor = self.ucs_version.split('.')[-1] + self.ucs_version = self.configRegistry['version/version'] + self.patchlevel = self.configRegistry['version/patchlevel'] + self.security_patchlevel = self.configRegistry['version/security-patchlevel'] + self.version_major, self.version_minor = self.ucs_version.split('.')[0:2] # should hotfixes be used - self.hotfixes = self.configRegistry.get( 'repository/online/hotfixes', 'no' ).lower() in ( 'true', 'yes' ) + self.hotfixes = self.ucr_bool('repository/online/hotfixes', False) - # check availability of the repository server - try: - socket.gethostbyname(self.repository_server) - self.nameserver_available=True - except socket.gaierror: - self.nameserver_available=False + self.config_repository() - # check for prefix on repository server (if the repository server is reachable) - try: - if not self.repository_prefix and self.net_path_exists( '/univention-repository/' ): - self.repository_prefix = 'univention-repository' - except: - self.repository_prefix = '' + def config_repository( self ): + """ Retrieve configuration to access repository. Overridden in UniventionMirror. """ + self.online_repository = self.ucr_bool('repository/online', True) + self.repositoryFromRegistry(self.configRegistry, 'repository/online') - def net_path_exists (self, path, server='', port='', prefix='', username='', password='', debug=False): + def net_path_exists (self, path, repository=None, debug=False): + """ Check if a path on the repository(defaults to self) exists. + >>> uup.net_path_exists('/') + True + >>> uup.net_path_exists('/foo',debug=True) + # The site http://apt.knut.univention.de:80/foo was not found + False + """ # path MUST NOT contain the schema and hostname - proxy_headers = self.open_connection(server=server, port=port) - if server: #if we use a diffrent server we should also use a diffrent prefix - if prefix: - site = '%s/%s/%s' % (self.proxy_prefix, prefix, path) - else: - site = '%s/%s' % (self.proxy_prefix, path) - else: - site = '%s/%s/%s' % (self.proxy_prefix, self.repository_prefix, path) - - replace_slash = re.compile ('[/]{2,}') - site = replace_slash.sub ('/', site) - if not site.startswith ('http://') and proxy_headers != None: - site = 'http://%s' % site - - if proxy_headers != None: - self.connection.putrequest('GET', site, skip_accept_encoding=1) - else: - self.connection.putrequest('GET', site) - - - if username and password: - auth = 'Basic ' + string.strip(base64.encodestring(username + ':' + password)) - self.connection.putheader('Authorization', auth) - - if proxy_headers != None: - for k, v in proxy_headers.items (): - self.connection.putheader (k, v) - self.connection.endheaders () - response = self.connection.getresponse() - response.read() - - if response.status == 200: - self.close_connection() + ''' check if 'path' exists on http://[username:password@]server[:port]/[prefix/]path ''' + if repository == None: + repository = self.repository() + #return repository.exists(path.lstrip('/')) + (response,data) = repository.access(path) + if response.status == httplib.OK: return True if debug: - if response.status == 404: - print '# The site http://%s%s was not found' % (server, site) - elif response.status == 401: + url = '%s%s' % (repository, path.lstrip('/')) + if response.status == httplib.NOT_FOUND: + print '# The site %s was not found' % url + elif response.status == httplib.UNAUTHORIZED: if username and password: - print '# Authentication failure for http://%s:%s@%s%s' % (username, password, server, site) + print '# Authentication failure for %s' % url else: - print '# Username and password are requiered for http://%s%s' % (server, site) + print '# Username and password are requiered for %s' % url else: - print '# The http error code (%d) was returned for the site http://%s%s' % (response.status, server, site) + print '# The http error code (%d) was returned for the site %s' % (response.status, url) - self.close_connection() return False def get_next_version( self, version ): - '''Check if a new patchlevel, minor or major release is available for the given version''' + '''Check if a new patchlevel, minor or major release is available for the given version + + >>> uup.get_next_version( UCS_Version( (2,1,2) ) ) + '2.2-0' + >>> uup.get_next_version( UCS_Version( (2,2,0) ) ) + '2.2-1' + >>> uup.get_next_version( UCS_Version( (99,99,99) ) ) + >>> + ''' if self.net_path_exists( '%d.%d/maintained/%d.%d-%d/' % ( version.major, version.minor, version.major, version.minor, version.patchlevel + 1 ) ): #check for x.y-(z+1) return '%d.%d-%d' % ( version.major, version.minor, version.patchlevel + 1 ) elif self.net_path_exists( '%d.%d/maintained/%d.%d-0/' % ( version.major, version.minor + 1, version.major, version.minor + 1 ) ): #check for x.(y+1)-0 @@ -267,33 +338,45 @@ return None def release_update_available( self, ucs_version = None ): - '''Check if an update is available for the ucs_version''' + '''Check if an update is available for the ucs_version + >>> uup.release_update_available( '2.1-2' ) + '2.2-0' + >>> uup.release_update_available( '2.2-0' ) + '2.2-1' + ''' if not ucs_version: return self.get_next_version( UCS_Version( ( self.version_major, self.version_minor, self.patchlevel ) ) ) return self.get_next_version( UCS_Version( ucs_version ) ) def security_update_temporary_sources_list(self): - '''Create a list of debian repository statements for the next security update''' + '''Create a list of debian repository statements for the next security update + >>> uup.security_update_temporary_sources_list() + ['deb http://apt.knut.univention.de:80/2.3/maintained/ sec1/all/', 'deb http://apt.knut.univention.de:80/2.3/maintained/ sec1/extern/', 'deb http://apt.knut.univention.de:80/2.3/maintained/ sec1/i386/'] + ''' sources_list = [] - for part in self.parts: + repConf = { 'server':str(self.repository()), 'version':self.ucs_version } + for repConf['part'] in self.parts: # for example: http://apt.univention.de/2.0/maintained/ - path='/%s/%s/' % (self.ucs_version, part) + path = '/%(version)s/%(part)s/' % repConf if not self.net_path_exists(path): continue - next_security_version = int(self.security_patchlevel) + 1 - path='/%s/%s/sec%s/' % (self.ucs_version, part, next_security_version) + repConf['sec'] = next_security_version = int(self.security_patchlevel) + 1 + path = '/%(version)s/%(part)s/sec%(sec)d/' % repConf if self.net_path_exists(path): - for arch in ['all', 'extern'] + self.architectures: - path='/%s/%s/sec%s/%s/' % (self.ucs_version, part, next_security_version, arch) + for repConf['arch'] in ['all', 'extern'] + self.architectures: + path = '/%(version)s/%(part)s/sec%(sec)d/%(arch)s/' % repConf if self.net_path_exists(path): - sources_list.append('deb http://%s/%s/%s/%s/ sec%s/%s/' % (self.repository_server, self.repository_prefix, self.ucs_version, part, next_security_version, arch)) + sources_list.append('deb %(server)s%(version)s/%(part)s/ sec%(sec)d/%(arch)s/' % repConf) return sources_list def security_update_available(self): - '''Check for the security version for the current version''' + '''Check for the security version for the current version. + >>> uup.security_update_available() + 'sec1' + ''' for part in self.parts: # for example: http://apt.univention.de/2.0/maintained/ path='/%s/%s/' % (self.ucs_version, part) @@ -308,20 +391,30 @@ return False def get_ucs_version(self): + '''Return current UCS version (major.minor-patchlevel). + >>> uup.get_ucs_version() + '2.3-0' + ''' return '%s-%s' % (self.ucs_version, self.patchlevel) def get_components(self): - '''Retrieve all enabled components from registry as list''' + '''Retrieve all enabled components from registry as list. + >>> uup.get_components() + ['ucsschool'] + ''' components = [] for key in self.configRegistry.keys(): if key.startswith('repository/online/component/'): component_part = key.split('repository/online/component/')[1] - if component_part.find('/') == -1 and self.configRegistry[key].lower() in [ 'true', 'yes', 'enabled', '1']: + if component_part.find('/') == -1 and self.ucr_bool(key, False): components.append(component_part) return components def get_all_components(self): - '''Retrieve all configured components from registry as list''' + '''Retrieve all configured components from registry as list. + >>> uup.get_all_components() + ['udm', 'ucsschool'] + ''' components = [] for key in self.configRegistry.keys(): if key.startswith('repository/online/component/'): @@ -331,7 +424,10 @@ return components def get_component(self, name): - '''Retrieve named component from registry as hash''' + '''Retrieve named component from registry as hash + >>> uup.get_component('udm') + {'description': 'udm?', 'activated': 'disabled', 'name': 'udm', 'server': 'apt.knut.univention.de'} + ''' component = {} for key in self.configRegistry.keys(): component['activated'] = self.configRegistry['repository/online/component/%s' % name] @@ -346,7 +442,11 @@ '''Return a string of Debian repository statements for all UCS versions between start and end. For dists=True, additional entries for the parts below dists/ are also added. - For clean=True, additional clean statements are added if online/repository/clean is enabled.''' + For clean=True, additional clean statements are added if online/repository/clean is enabled. + + >>> uup.print_version_repositories(clean=True,dists=True,start=UCS_Version('2.2-0'),end=UCS_Version('2.2-1')).split(chr(10)) + ['deb http://apt.knut.univention.de:80/2.2/maintained/2.2-0 dists/univention/main/binary-i386/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ 2.2-0/all/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ 2.2-0/extern/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ 2.2-0/i386/', 'clean http://apt.knut.univention.de:80/2.2/maintained/2.2-0/', '', 'deb http://apt.knut.univention.de:80/2.2/maintained/ 2.2-1/all/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ 2.2-1/extern/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ 2.2-1/i386/', 'clean http://apt.knut.univention.de:80/2.2/maintained/2.2-1/', '', ''] + ''' repos = '' if not self.online_repository: return repos @@ -360,56 +460,48 @@ end = UCS_Version( ( self.version_major, self.version_minor, self.patchlevel ) ) if clean: - clean = self.configRegistry.get( 'online/repository/clean', False ) + clean = self.ucr_bool('online/repository/clean', False) + repConf = { 'server':str(self.repository()) } while start <= end: + repConf.update(major=start.major, minor=start.minor, patch=start.patchlevel) # for example: http://apt.univention.de/2.0/ - path='/%s.%s/' % (start.major, start.minor) + path='/%(major)d.%(minor)d/' % repConf if not self.net_path_exists( path ): start.minor += 1 continue if dists: - path_base = '/%s.%s/maintained/%s.%s-%s' % ( start.major, start.minor, start.major, start.minor, start.patchlevel ) + path_base = '%(major)d.%(minor)d/maintained/%(major)d.%(minor)d-%(patch)d' % repConf if self.net_path_exists( '%s/dists/' % path_base ): for arch in self.architectures: path = '%s/dists/univention/main/binary-%s/' % ( path_base, arch ) if self.net_path_exists( path ): - if self.repository_prefix: - repos += 'deb http://%s/%s%s dists/univention/main/binary-%s/\n' % ( self.repository_server, self.repository_prefix, path_base, arch ) - else: - repos += 'deb http://%s%s dists/univention/main/binary-%s/\n' % ( self.repository_server, path_base, arch ) - for part in self.parts: + repos += 'deb %s%s dists/univention/main/binary-%s/\n' % ( self.repository(), path_base, arch ) + for repConf['part'] in self.parts: # for example: http://apt.univention.de/2.0/maintained/ - path='/%s.%s/%s/' % (start.major, start.minor, part) + path='/%(major)d.%(minor)d/%(part)s/' % repConf if not self.net_path_exists(path): continue patch_inc = UCS_Version( ( start.major, start.minor, start.patchlevel ) ) # as long as we do just increase the patch level ... while patch_inc.major == start.major and patch_inc.minor == start.minor: - path='/%s.%s/%s/%s.%s-%s/' % ( patch_inc.major, patch_inc.minor, part, patch_inc.major, patch_inc.minor, patch_inc.patchlevel ) + repConf.update(major=patch_inc.major, minor=patch_inc.minor, patch=patch_inc.patchlevel) + path='/%(major)d.%(minor)d/%(part)s/%(major)d.%(minor)d-%(patch)d/' % repConf if not self.net_path_exists(path): break # the helper variable printed is to be used to print a blank line at the end of a block printed = False - for arch in ['all', 'extern'] + self.architectures: + for repConf['arch'] in ['all', 'extern'] + self.architectures: # for example: http://apt.univention.de/2.0/maintained/2.0-1 - path='/%s.%s/%s/%s.%s-%s/%s/Packages.gz' % ( patch_inc.major, patch_inc.minor, part, patch_inc.major, patch_inc.minor, patch_inc.patchlevel, arch ) + path='/%(major)d.%(minor)d/%(part)s/%(major)d.%(minor)d-%(patch)d/%(arch)s/Packages.gz' % repConf if not self.net_path_exists(path): continue printed = True - if self.repository_prefix: - path = 'http://%s/%s/%s.%s/%s/' % ( self.repository_server, self.repository_prefix, patch_inc.major, patch_inc.minor, part ) - else: - path = 'http://%s/%s.%s/%s/' % ( self.repository_server, patch_inc.major, patch_inc.minor, part ) - repos += 'deb %s %s.%s-%s/%s/\n' % ( path, patch_inc.major, patch_inc.minor , patch_inc.patchlevel, arch) + repos += 'deb %(server)s%(major)d.%(minor)d/%(part)s/ %(major)d.%(minor)d-%(patch)d/%(arch)s/\n' % repConf if clean: - if self.repository_prefix: - path = 'http://%s/%s/%s.%s/%s/' % ( self.repository_server, self.repository_prefix, patch_inc.major, patch_inc.minor, part ) - else: - path = 'http://%s/%s.%s/%s/' % ( self.repository_server, patch_inc.major, patch_inc.minor, part ) - repos += 'clean %s/%s.%s-%s/\n' % ( path, patch_inc.major, patch_inc.minor , patch_inc.patchlevel ) + repos += 'clean %(server)s%(major)d.%(minor)d/%(part)s/%(major)d.%(minor)d-%(patch)d/\n' % repConf if printed: repos += '\n' @@ -433,7 +525,11 @@ updates for UCS versions between start and end. For clean=True, additional clean statements are added if online/repository/clean is enabled. For all_security_updates=True, all available instead of all needed - statements for security updates are returned.''' + statements for security updates are returned. + + >>> uup.print_security_repositories(clean=True,start=UCS_Version('2.2-0'),end=UCS_Version('2.2-1')).split(chr(10)) + ['deb http://apt.knut.univention.de:80/2.2/maintained/ sec1/all/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec1/extern/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec1/i386/', 'clean http://apt.knut.univention.de:80/2.2/maintained/sec1/', '', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec2/all/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec2/extern/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec2/i386/', 'clean http://apt.knut.univention.de:80/2.2/maintained/sec2/', '', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec3/all/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec3/extern/', 'deb http://apt.knut.univention.de:80/2.2/maintained/ sec3/i386/', 'clean http://apt.knut.univention.de:80/2.2/maintained/sec3/', '', ''] + ''' repos = '' if not self.online_repository: return repos @@ -449,39 +545,36 @@ if clean: clean = self.configRegistry.get( 'online/repository/clean', False ) - while start <=end: + repConf = { 'server':str(self.repository()) } + while start <= end: + repConf.update({ 'major':start.major, 'minor':start.minor, 'patch':start.patchlevel }) # check for the security version for the current version - for part in self.parts: + for repConf['part'] in self.parts: # for example: http://apt.univention.de/2.0/maintained/ - path='/%d.%d/%s/' % ( start.major, start.minor, part) + path='/%(major)d.%(minor)d/%(part)s/' % repConf if not self.net_path_exists(path): continue # I think we won't release more than 100 security releases for one UCS release ... for p in range(1, 100): + repConf['sec'] = p # the sources.list should just contain the already installed seucurity repos; the mirror list should contain all available security repos if not all_security_updates and start.major == int( self.version_major ) and start.minor == int( self.version_minor ) and p > int( self.security_patchlevel ): break - path='/%d.%d/%s/sec%s/' % ( start.major, start.minor, part, p ) + path='/%(major)d.%(minor)d/%(part)s/sec%(sec)d/' % repConf if not self.net_path_exists(path): break printed = False - for arch in ['all', 'extern'] + self.architectures: + for repConf['arch'] in ['all', 'extern'] + self.architectures: # for example: http://apt.univention.de/2.0/maintained/sec1 - path='/%d.%d/%s/sec%s/%s/Packages.gz' % (start.major, start.minor, part, p, arch ) + path='/%(major)d.%(minor)d/%(part)s/sec%(sec)s/%(arch)s/Packages.gz' % repConf if not self.net_path_exists(path): continue printed = True - if self.repository_prefix: - repos += 'deb http://%s/%s/%d.%d/%s/ sec%s/%s/\n' % ( self.repository_server, self.repository_prefix, start.major, start.minor, part, p, arch) - else: - repos += 'deb http://%s/%d.%d/%s/ sec%s/%s/\n' % ( self.repository_server, start.major, start.minor, part, p, arch) + repos += 'deb %(server)s%(major)d.%(minor)d/%(part)s/ sec%(sec)d/%(arch)s/\n' % repConf if clean: - if self.repository_prefix: - repos += 'clean http://%s/%s/%d.%d/%s/sec%s/%s/\n' % ( self.repository_server, self.repository_prefix, start.major, start.minor, part, p ) - else: - repos += 'clean http://%s/%d.%d/%s/sec%s/%s/\n' % ( self.repository_server, start.major, start.minor, part, p ) + repos += 'clean %(server)s%(major)d.%(minor)d/%(part)s/sec%(sec)d/\n' % repConf if printed: repos += '\n' printed = False @@ -489,15 +582,12 @@ # check for hotfixes? if not self.hotfixes: continue - path = '/%s/%s/hotfixes/' % ( self.ucs_version, part ) + path = '/%(major)d.%(minor)d/%(part)s/hotfixes/' % repConf if self.net_path_exists( path ): - for arch in [ 'all', 'extern' ] + self.architectures: - path='/%s/%s/hotfixes/%s/Packages.gz' % ( self.ucs_version, part, arch ) + for repConf['arch'] in [ 'all', 'extern' ] + self.architectures: + path='/%(major)d.%(minor)d/%(part)s/hotfixes/%(arch)s/Packages.gz' % repConf if self.net_path_exists( path ): - if self.repository_prefix: - repos += 'deb http://%s/%s/%s/%s/ hotfixes/%s/\n' % ( self.repository_server, self.repository_prefix, self.ucs_version, part, arch ) - else: - repos += 'deb http://%s/%s/%s/ hotfixes/%s/\n' % ( self.repository_server, self.ucs_version, part, arch ) + repos += 'deb %(server)s%(major)d.%(minor)d/%(part)s/ hotfixes/%(arch)s/\n' % repConf start.minor += 1 # is there a minor version update @@ -515,7 +605,12 @@ def print_component_repositories( self, clean = False ): '''Return a string of Debian repository statements for all enabled components. - For clean=True, additional clean statements are added if online/repository/clean is enabled.''' + For clean=True, additional clean statements are added if online/repository/clean is enabled. + + >>> uup.print_component_repositories(clean=True).split(chr(10)) + # The site http://apt.knut.univention.de:80/2.3/maintained/component/ucsschool/Packages.gz was not found + ['deb http://apt.knut.univention.de:80/2.3/maintained/component ucsschool/all/', 'deb http://apt.knut.univention.de:80/2.3/maintained/component ucsschool/extern/', 'deb http://apt.knut.univention.de:80/2.3/maintained/component ucsschool/i386/', '', ''] + ''' repos = '' if not self.online_repository: return repos @@ -523,87 +618,82 @@ version_part_left = int( self.version_major ) version_part_right = int( self.version_minor ) if clean: - clean = self.configRegistry.get( 'online/repository/clean', False ) + clean = self.ucr_bool('online/repository/clean', False) components = [] for key in self.configRegistry.keys(): if key.startswith('repository/online/component/'): component_part = key.split('repository/online/component/')[1] - if component_part.find('/') == -1 and self.configRegistry[key].lower() in [ 'true', 'yes', 'enabled', '1']: + if component_part.find('/') == -1 and self.ucr_bool(key, False): components.append(component_part) for component in components: if not self.is_repository_server: - repository_server = self.configRegistry.get('repository/online/component/%s/server' % component, self.repository_server) + config = RepositoryConfig() + config.repositoryFromRegistry(self.configRegistry, 'repository/online/component/%s' % component) + repository = config.repository() else: - repository_server = self.repository_server - repository_port = self.configRegistry.get('repository/online/component/%s/port' % component, self.repository_port) - prefix_var = 'repository/online/component/%s/prefix' % component - repository_prefix = self.configRegistry.get( 'repository/online/component/%s/prefix' % component, '' ) + repository = self.repository() versions = self.configRegistry.get('repository/online/component/%s/version' % component, self.ucs_version).split(',') parts = self.configRegistry.get('repository/online/component/%s/parts' % component, 'maintained').split(',') - username = self.configRegistry.get('repository/online/component/%s/username' % component, None) - password = self.configRegistry.get('repository/online/component/%s/password' % component, None) if clean: - clean = self.configRegistry.get( 'repository/online/component/%s/clean' % component, False ) + clean = self.ucr_bool('repository/online/component/%s/clean' % component, False) - # allow None as a component prefix - if repository_prefix.lower() == 'none': - repository_prefix = '' - elif not repository_prefix: - # check for prefix on component repository server (if the repository server is reachable) - try: - if self.net_path_exists( '/univention-repository/', server = repository_server, port = repository_port, username = username, password = password ): - repository_prefix = 'univention-repository' - elif self.net_path_exists( '/%s/' % self.repository_prefix, server = repository_server, port = repository_port, username = username, password = password ): - repository_prefix = self.repository_prefix - except: - repository_prefix = '' - + repConf = { 'server':str(repository), 'component':component } for version in versions: if version == 'current': version = self.ucs_version - for part in parts: - auth_string = '' - if username and password: - auth_string = '%s:%s@' % (username, password) + repConf['version'] = version + for repConf['part'] in parts: #2.0/maintained/component/ - path = '/%s/%s/component/%s/' % ( version, part, component ) - if not self.net_path_exists(path, server=repository_server, port=repository_port, prefix=repository_prefix, username=username, password=password, debug=True): + path = '/%(version)s/%(part)s/component/%(component)s/' % repConf + if not self.net_path_exists(path, repository, debug=True): continue printed = False # support a diffrent repository - path = '/%s/%s/component/%s/Packages.gz' % ( version, part, component ) - if self.net_path_exists(path, server=repository_server, port=repository_port, prefix=repository_prefix, username=username, password=password, debug=True): - if repository_prefix: - path = 'http://%s%s/%s/%s/%s/component/%s/' % ( auth_string, repository_server, repository_prefix, version, part, component) - else: - path = 'http://%s%s/%s/%s/component/%s/' % ( auth_string, repository_server, version, part, component) - repos += 'deb %s ./ \n' % path + path = '/%(version)s/%(part)s/component/%(component)s/Packages.gz' % repConf + if self.net_path_exists(path, repository, debug=True): + path = '%(server)s%(version)s/%(part)s/component/%(component)s/' % repConf + repos += 'deb %s ./\n' % path if clean: repos += 'clean %s\n' % path printed = True else: - for arch in ['all', 'extern'] + self.architectures: - path = '/%s/%s/component/%s/%s/' % ( version, part, component, arch ) - if not self.net_path_exists(path, server=repository_server, port=repository_port, prefix=repository_prefix, username=username, password=password, debug=True): + for repConf['arch'] in ['all', 'extern'] + self.architectures: + path = '/%(version)s/%(part)s/component/%(component)s/%(arch)s/' % repConf + if not self.net_path_exists(path, repository, debug=True): continue printed = True - if repository_prefix: - path = 'http://%s%s/%s/%s/%s/' % ( auth_string, repository_server, repository_prefix, version, part ) - else: - path = 'http://%s%s/%s/%s/' % ( auth_string, repository_server, version, part ) - repos += 'deb %scomponent %s/%s/\n' % ( path, component, arch ) + repos += 'deb %(server)s%(version)s/%(part)s/component %(component)s/%(arch)s/\n' % repConf if clean: - if repository_prefix: - path = 'http://%s%s/%s/%s/%s/' % ( auth_string, repository_server, repository_prefix, version, part ) - else: - path = 'http://%s%s/%s/%s/' % ( auth_string, repository_server, version, part ) - repos += 'clean %s/component/%s/\n' % ( path, component ) + repos += 'clean %(server)s%(version)s/%(part)s/component/%(component)s/\n' % repConf if printed: repos += '\n' printed = False return repos + +if __name__ == '__main__': + # repository_mirror_basepath="/var/lib/univention-repository" + # repository_mirror="yes" + # repository_online_component_ucsschool_description="UCS School" + # repository_online_component_ucsschool_server="apt.knut.univention.de" + # repository_online_component_ucsschool="enabled" + # repository_online_component_udm_description="udm?" + # repository_online_component_udm_server="apt.knut.univention.de" + # repository_online_component_udm="disabled" + # repository_online_hotfixes="no" + # repository_online_maintained="yes" + # repository_online_port="80" + # repository_online_server="apt.knut.univention.de" + # repository_online_unmaintained="no" + # repository_online="yes" + # local_repository="" + # version_patchlevel="0 + # version_security_patchlevel="0 + # version_version="2.3" + uup = UniventionUpdater() + import doctest + doctest.testmod()