|
Lines 562-593
class Application(object):
|
Link Here
|
|---|
|
|---|
| 562 |
except: |
562 |
except: |
| 563 |
MODULE.error('JSON malformatted: "%s"' % content) |
563 |
MODULE.error('JSON malformatted: "%s"' % content) |
| 564 |
raise |
564 |
raise |
| 565 |
files_to_download = [] |
565 |
files_to_download = {} |
| 566 |
files_in_json_file = [] |
566 |
files_in_json_file = [] |
| 567 |
for appname, appinfo in json_apps.iteritems(): |
567 |
for appname, appinfo in json_apps.iteritems(): |
| 568 |
for appfile, appfileinfo in appinfo.iteritems(): |
568 |
for appfile, appfileinfo in appinfo.iteritems(): |
| 569 |
filename = '%s.%s' % (appname, appfile) |
569 |
filename = '%s.%s' % (appname, appfile) |
| 570 |
remote_md5sum = appfileinfo['md5'] |
|
|
| 571 |
remote_url = appfileinfo['url'] |
| 572 |
# compare with local cache |
570 |
# compare with local cache |
| 573 |
cached_filename = os.path.join(CACHE_DIR, filename) |
571 |
cached_filename = os.path.join(CACHE_DIR, filename) |
| 574 |
files_in_json_file.append(cached_filename) |
572 |
files_in_json_file.append(cached_filename) |
| 575 |
local_md5sum = None |
|
|
| 576 |
m = md5() |
| 577 |
if os.path.exists(cached_filename): |
573 |
if os.path.exists(cached_filename): |
|
|
574 |
m = md5() |
| 578 |
with open(cached_filename, 'r') as f: |
575 |
with open(cached_filename, 'r') as f: |
| 579 |
m.update(f.read()) |
576 |
m.update(f.read()) |
| 580 |
local_md5sum = m.hexdigest() |
577 |
if m.hexdigest() != appfileinfo['md5']: |
| 581 |
if remote_md5sum != local_md5sum: |
578 |
# ask to re-download this file |
| 582 |
# ask to re-download this file |
579 |
filename_url = appfileinfo['url'] |
| 583 |
files_to_download.append((remote_url, filename)) |
580 |
parts = urlsplit(filename_url) |
| 584 |
something_changed = True |
581 |
files_to_download.setdefault(parts.netloc, set()).add((filename_url, cached_filename)) |
|
|
582 |
something_changed = True |
| 585 |
# remove those files that apparently do not exist on server anymore |
583 |
# remove those files that apparently do not exist on server anymore |
| 586 |
for cached_filename in glob(os.path.join(CACHE_DIR, '*')): |
584 |
for cached_filename in glob(os.path.join(CACHE_DIR, '*')): |
| 587 |
if cached_filename not in files_in_json_file: |
585 |
if cached_filename not in files_in_json_file: |
| 588 |
MODULE.info('Deleting obsolete %s' % cached_filename) |
586 |
MODULE.info('Deleting obsolete %s' % cached_filename) |
| 589 |
something_changed = True |
587 |
something_changed = True |
| 590 |
os.unlink(cached_filename) |
588 |
os.unlink(cached_filename) |
|
|
589 |
|
| 591 |
def _download(url, dest): |
590 |
def _download(url, dest): |
| 592 |
MODULE.info('Downloading %s to %s' % (url, dest)) |
591 |
MODULE.info('Downloading %s to %s' % (url, dest)) |
| 593 |
try: |
592 |
try: |
|
Lines 597-617
class Application(object):
|
Link Here
|
|---|
|
|---|
| 597 |
else: |
596 |
else: |
| 598 |
with open(dest, 'wb') as f: |
597 |
with open(dest, 'wb') as f: |
| 599 |
f.write(urlcontent.read()) |
598 |
f.write(urlcontent.read()) |
| 600 |
threads = [] |
|
|
| 601 |
for filename_url, filename in files_to_download: |
| 602 |
# dont forget to quote: 'foo & bar.ini' -> 'foo%20&%20bar.ini' |
| 603 |
# but dont quote https:// -> https%3A// |
| 604 |
parts = list(urlsplit(filename_url)) |
| 605 |
parts[2] = urllib2.quote(parts[2]) # 0 -> scheme, 1 -> netloc, 2 -> path |
| 606 |
filename_url = urlunsplit(parts) |
| 607 |
|
599 |
|
| 608 |
cached_filename = os.path.join(CACHE_DIR, filename) |
600 |
def _download_from_host(jobs): |
|
|
601 |
for filename_url, filename in jobs: |
| 602 |
_download(filename_url, filename) |
| 609 |
|
603 |
|
| 610 |
thread = Thread(target=_download, args=(filename_url, cached_filename)) |
604 |
threads = [Thread(target=_download_from_host, args=(jobs,)) for jobs in files_to_download.values()] |
|
|
605 |
for thread in threads: |
| 611 |
thread.start() |
606 |
thread.start() |
| 612 |
threads.append(thread) |
|
|
| 613 |
for thread in threads: |
607 |
for thread in threads: |
| 614 |
thread.join() |
608 |
thread.join() |
|
|
609 |
|
| 615 |
if something_changed: |
610 |
if something_changed: |
| 616 |
# some variables could change apps.xml |
611 |
# some variables could change apps.xml |
| 617 |
# e.g. Name, Description |
612 |
# e.g. Name, Description |