Attachment #5903 for bug #32935

View | Details | Raw Unified | Return to bug 32935
Collapse All | Expand All




		except:
			MODULE.error('JSON malformatted: "%s"' % content)
			raise
		files_to_download = {}
		files_in_json_file = []
		for appname, appinfo in json_apps.iteritems():
			for appfile, appfileinfo in appinfo.iteritems():
				filename = '%s.%s' % (appname, appfile)
				remote_md5sum = appfileinfo['md5']
				remote_url = appfileinfo['url']
				# compare with local cache
				cached_filename = os.path.join(CACHE_DIR, filename)
				files_in_json_file.append(cached_filename)
				local_md5sum = None
				m = md5()
				if os.path.exists(cached_filename):
					m = md5()
					with open(cached_filename, 'r') as f:
						m.update(f.read())
					if m.hexdigest() != appfileinfo['md5']:
						# ask to re-download this file
						filename_url = appfileinfo['url']
						parts = urlsplit(filename_url)
						files_to_download.setdefault(parts.netloc, set()).add((filename_url, cached_filename))
						something_changed = True
		# remove those files that apparently do not exist on server anymore
		for cached_filename in glob(os.path.join(CACHE_DIR, '*')):
			if cached_filename not in files_in_json_file:
				MODULE.info('Deleting obsolete %s' % cached_filename)
				something_changed = True
				os.unlink(cached_filename)

		def _download(url, dest):
			MODULE.info('Downloading %s to %s' % (url, dest))
			try:

			else:
				with open(dest, 'wb') as f:
					f.write(urlcontent.read())
		threads = []
		for filename_url, filename in files_to_download:
			# dont forget to quote: 'foo & bar.ini' -> 'foo%20&%20bar.ini'
			# but dont quote https:// -> https%3A//
			parts = list(urlsplit(filename_url))
			parts[2] = urllib2.quote(parts[2]) # 0 -> scheme, 1 -> netloc, 2 -> path
			filename_url = urlunsplit(parts)

		def _download_from_host(jobs):
			for filename_url, filename in jobs:
				_download(filename_url, filename)

		threads = [Thread(target=_download_from_host, args=(jobs,)) for jobs in files_to_download.values()]
		for thread in threads:
			thread.start()
			threads.append(thread)
		for thread in threads:
			thread.join()

		if something_changed:
			# some variables could change apps.xml
			# e.g. Name, Description

Lines 70-76 def rename_app(old_id, new_id, component_manager, package_manager): Link Here

(-)a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/util.py (-2 / +2 lines)
70	app.tell_ldap(component_manager.ucr, package_manager, inform_about_error=False)	70	app.tell_ldap(component_manager.ucr, package_manager, inform_about_error=False)
71		71
72	def get_hosts(module, lo, ucr=None):	72	def get_hosts(module, lo, ucr=None):
73	hosts = module.lookup(None, lo, None)	73	hosts = module.lookup(None, lo, None)
74	hostnames = []	74	hostnames = []
75	if ucr is not None:	75	if ucr is not None:
76	local_hostname = ucr.get('hostname')	76	local_hostname = ucr.get('hostname')
Lines 114-120 def install_opener(ucr): Link Here
114	def urlopen(request):	114	def urlopen(request):
115	# use this in __init__ and app_center	115	# use this in __init__ and app_center
116	# to have the proxy handler installed globally	116	# to have the proxy handler installed globally
117	return urllib2.urlopen(request)	117	return urllib2.urlopen(request, timeout=60)
118		118
119	def get_current_ram_available():	119	def get_current_ram_available():
120	''' Returns RAM currently available in MB, excluding Swap '''	120	''' Returns RAM currently available in MB, excluding Swap '''

Return to bug 32935

Lines 562-593 class Application(object): Link Here

(-)a/branches/ucs-3.2/ucs-3.2-2/management/univention-management-console-module-appcenter/umc/python/appcenter/app_center.py (-20 / +15 lines)
562	except:	562	except:
563	MODULE.error('JSON malformatted: "%s"' % content)	563	MODULE.error('JSON malformatted: "%s"' % content)
564	raise	564	raise
565	files_to_download = []	565	files_to_download = {}
566	files_in_json_file = []	566	files_in_json_file = []
567	for appname, appinfo in json_apps.iteritems():	567	for appname, appinfo in json_apps.iteritems():
568	for appfile, appfileinfo in appinfo.iteritems():	568	for appfile, appfileinfo in appinfo.iteritems():
569	filename = '%s.%s' % (appname, appfile)	569	filename = '%s.%s' % (appname, appfile)
570	remote_md5sum = appfileinfo['md5']
571	remote_url = appfileinfo['url']
572	# compare with local cache	570	# compare with local cache
573	cached_filename = os.path.join(CACHE_DIR, filename)	571	cached_filename = os.path.join(CACHE_DIR, filename)
574	files_in_json_file.append(cached_filename)	572	files_in_json_file.append(cached_filename)
575	local_md5sum = None
576	m = md5()
577	if os.path.exists(cached_filename):	573	if os.path.exists(cached_filename):
		574	m = md5()
578	with open(cached_filename, 'r') as f:	575	with open(cached_filename, 'r') as f:
579	m.update(f.read())	576	m.update(f.read())
580	local_md5sum = m.hexdigest()	577	if m.hexdigest() != appfileinfo['md5']:
581	if remote_md5sum != local_md5sum:	578	# ask to re-download this file
582	# ask to re-download this file	579	filename_url = appfileinfo['url']
583	files_to_download.append((remote_url, filename))	580	parts = urlsplit(filename_url)
584	something_changed = True	581	files_to_download.setdefault(parts.netloc, set()).add((filename_url, cached_filename))
		582	something_changed = True
585	# remove those files that apparently do not exist on server anymore	583	# remove those files that apparently do not exist on server anymore
586	for cached_filename in glob(os.path.join(CACHE_DIR, '*')):	584	for cached_filename in glob(os.path.join(CACHE_DIR, '*')):
587	if cached_filename not in files_in_json_file:	585	if cached_filename not in files_in_json_file:
588	MODULE.info('Deleting obsolete %s' % cached_filename)	586	MODULE.info('Deleting obsolete %s' % cached_filename)
589	something_changed = True	587	something_changed = True
590	os.unlink(cached_filename)	588	os.unlink(cached_filename)
		589
591	def _download(url, dest):	590	def _download(url, dest):
592	MODULE.info('Downloading %s to %s' % (url, dest))	591	MODULE.info('Downloading %s to %s' % (url, dest))
593	try:	592	try:
Lines 597-617 class Application(object): Link Here
597	else:	596	else:
598	with open(dest, 'wb') as f:	597	with open(dest, 'wb') as f:
599	f.write(urlcontent.read())	598	f.write(urlcontent.read())
600	threads = []
601	for filename_url, filename in files_to_download:
602	# dont forget to quote: 'foo & bar.ini' -> 'foo%20&%20bar.ini'
603	# but dont quote https:// -> https%3A//
604	parts = list(urlsplit(filename_url))
605	parts[2] = urllib2.quote(parts[2]) # 0 -> scheme, 1 -> netloc, 2 -> path
606	filename_url = urlunsplit(parts)
607		599
608	cached_filename = os.path.join(CACHE_DIR, filename)	600	def _download_from_host(jobs):
		601	for filename_url, filename in jobs:
		602	_download(filename_url, filename)
609		603
610	thread = Thread(target=_download, args=(filename_url, cached_filename))	604	threads = [Thread(target=_download_from_host, args=(jobs,)) for jobs in files_to_download.values()]
		605	for thread in threads:
611	thread.start()	606	thread.start()
612	threads.append(thread)
613	for thread in threads:	607	for thread in threads:
614	thread.join()	608	thread.join()
		609
615	if something_changed:	610	if something_changed:
616	# some variables could change apps.xml	611	# some variables could change apps.xml
617	# e.g. Name, Description	612	# e.g. Name, Description