diff --git a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py index 2904618..ea21ae4 100644 --- a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py +++ b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py @@ -562,32 +562,31 @@ class Application(object): except: MODULE.error('JSON malformatted: "%s"' % content) raise - files_to_download = [] + files_to_download = {} files_in_json_file = [] for appname, appinfo in json_apps.iteritems(): for appfile, appfileinfo in appinfo.iteritems(): filename = '%s.%s' % (appname, appfile) - remote_md5sum = appfileinfo['md5'] - remote_url = appfileinfo['url'] # compare with local cache cached_filename = os.path.join(CACHE_DIR, filename) files_in_json_file.append(cached_filename) - local_md5sum = None - m = md5() if os.path.exists(cached_filename): + m = md5() with open(cached_filename, 'r') as f: m.update(f.read()) - local_md5sum = m.hexdigest() - if remote_md5sum != local_md5sum: - # ask to re-download this file - files_to_download.append((remote_url, filename)) - something_changed = True + if m.hexdigest() != appfileinfo['md5']: + # ask to re-download this file + filename_url = appfileinfo['url'] + parts = urlsplit(filename_url) + files_to_download.setdefault(parts.netloc, set()).add((filename_url, cached_filename)) + something_changed = True # remove those files that apparently do not exist on server anymore for cached_filename in glob(os.path.join(CACHE_DIR, '*')): if cached_filename not in files_in_json_file: MODULE.info('Deleting obsolete %s' % cached_filename) something_changed = True os.unlink(cached_filename) + def _download(url, dest): MODULE.info('Downloading %s to %s' % (url, dest)) try: @@ -597,21 +596,17 @@ class Application(object): else: with open(dest, 'wb') as f: f.write(urlcontent.read()) - threads = [] - for filename_url, filename in files_to_download: - # dont forget to quote: 'foo & bar.ini' -> 'foo%20&%20bar.ini' - # but dont quote https:// -> https%3A// - parts = list(urlsplit(filename_url)) - parts[2] = urllib2.quote(parts[2]) # 0 -> scheme, 1 -> netloc, 2 -> path - filename_url = urlunsplit(parts) - cached_filename = os.path.join(CACHE_DIR, filename) + def _download_from_host(jobs): + for filename_url, filename in jobs: + _download(filename_url, filename) - thread = Thread(target=_download, args=(filename_url, cached_filename)) + threads = [Thread(target=_download_from_host, args=(jobs,)) for jobs in files_to_download.values()] + for thread in threads: thread.start() - threads.append(thread) for thread in threads: thread.join() + if something_changed: # some variables could change apps.xml # e.g. Name, Description diff --git a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py index 697997f..ad1e3e4 100644 --- a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py +++ b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py @@ -70,7 +70,7 @@ def rename_app(old_id, new_id, component_manager, package_manager): app.tell_ldap(component_manager.ucr, package_manager, inform_about_error=False) def get_hosts(module, lo, ucr=None): - hosts = module.lookup(None, lo, None) + hosts = module.lookup(None, lo, None) hostnames = [] if ucr is not None: local_hostname = ucr.get('hostname') @@ -114,7 +114,7 @@ def install_opener(ucr): def urlopen(request): # use this in __init__ and app_center # to have the proxy handler installed globally - return urllib2.urlopen(request) + return urllib2.urlopen(request, timeout=60) def get_current_ram_available(): ''' Returns RAM currently available in MB, excluding Swap '''