Attachment 5903 Details for Bug 32935

[patch] Limit download threads

32935_app-pipelining.diff (text/plain), 4.29 KB, created by Philipp Hahn on 2014-05-09 15:14 CEST

(hide)

Description:

Filename:

MIME Type:

Creator: Philipp Hahn

Created: 2014-05-09 15:14 CEST

Size: 4.29 KB

patch

obsolete

>diff --git a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py
>index 2904618..ea21ae4 100644
>--- a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py
>+++ b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py
>@@ -562,32 +562,31 @@ class Application(object):
> 		except:
> 			MODULE.error('JSON malformatted: "%s"' % content)
> 			raise
>-		files_to_download = []
>+		files_to_download = {}
> 		files_in_json_file = []
> 		for appname, appinfo in json_apps.iteritems():
> 			for appfile, appfileinfo in appinfo.iteritems():
> 				filename = '%s.%s' % (appname, appfile)
>-				remote_md5sum = appfileinfo['md5']
>-				remote_url = appfileinfo['url']
> 				# compare with local cache
> 				cached_filename = os.path.join(CACHE_DIR, filename)
> 				files_in_json_file.append(cached_filename)
>-				local_md5sum = None
>-				m = md5()
> 				if os.path.exists(cached_filename):
>+					m = md5()
> 					with open(cached_filename, 'r') as f:
> 						m.update(f.read())
>-						local_md5sum = m.hexdigest()
>-				if remote_md5sum != local_md5sum:
>-					# ask to re-download this file
>-					files_to_download.append((remote_url, filename))
>-					something_changed = True
>+					if m.hexdigest() != appfileinfo['md5']:
>+						# ask to re-download this file
>+						filename_url = appfileinfo['url']
>+						parts = urlsplit(filename_url)
>+						files_to_download.setdefault(parts.netloc, set()).add((filename_url, cached_filename))
>+						something_changed = True
> 		# remove those files that apparently do not exist on server anymore
> 		for cached_filename in glob(os.path.join(CACHE_DIR, '*')):
> 			if cached_filename not in files_in_json_file:
> 				MODULE.info('Deleting obsolete %s' % cached_filename)
> 				something_changed = True
> 				os.unlink(cached_filename)
>+
> 		def _download(url, dest):
> 			MODULE.info('Downloading %s to %s' % (url, dest))
> 			try:
>@@ -597,21 +596,17 @@ class Application(object):
> 			else:
> 				with open(dest, 'wb') as f:
> 					f.write(urlcontent.read())
>-		threads = []
>-		for filename_url, filename in files_to_download:
>-			# dont forget to quote: 'foo & bar.ini' -> 'foo%20&%20bar.ini'
>-			# but dont quote https:// -> https%3A//
>-			parts = list(urlsplit(filename_url))
>-			parts[2] = urllib2.quote(parts[2]) # 0 -> scheme, 1 -> netloc, 2 -> path
>-			filename_url = urlunsplit(parts)
> 
>-			cached_filename = os.path.join(CACHE_DIR, filename)
>+		def _download_from_host(jobs):
>+			for filename_url, filename in jobs:
>+				_download(filename_url, filename)
> 
>-			thread = Thread(target=_download, args=(filename_url, cached_filename))
>+		threads = [Thread(target=_download_from_host, args=(jobs,)) for jobs in files_to_download.values()]
>+		for thread in threads:
> 			thread.start()
>-			threads.append(thread)
> 		for thread in threads:
> 			thread.join()
>+
> 		if something_changed:
> 			# some variables could change apps.xml
> 			# e.g. Name, Description
>diff --git a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py
>index 697997f..ad1e3e4 100644
>--- a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py
>+++ b/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py
>@@ -70,7 +70,7 @@ def rename_app(old_id, new_id, component_manager, package_manager):
> 	app.tell_ldap(component_manager.ucr, package_manager, inform_about_error=False)
> 
> def get_hosts(module, lo, ucr=None):
>- 	hosts = module.lookup(None, lo, None)
>+	hosts = module.lookup(None, lo, None)
> 	hostnames = []
> 	if ucr is not None:
> 		local_hostname = ucr.get('hostname')
>@@ -114,7 +114,7 @@ def install_opener(ucr):
> def urlopen(request):
> 	# use this in __init__ and app_center
> 	# to have the proxy handler installed globally
>-	return urllib2.urlopen(request)
>+	return urllib2.urlopen(request, timeout=60)
> 
> def get_current_ram_available():
> 	''' Returns RAM currently available in MB, excluding Swap '''

Actions: View | Diff

Attachments on bug 32935: 5903 | 5908 | 5982