Univention Bugzilla – Attachment 4826 Details for
Bug 28893
Anpassung: announce_errata_update auf neues Errata-Verfahren
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
Meine (teilweise bereits besser) funktionierende Variante zum Vergleich
announce_errata.py (text/plain), 19.80 KB, created by
Philipp Hahn
on 2012-11-21 16:29:02 CET
(
hide
)
Description:
Meine (teilweise bereits besser) funktionierende Variante zum Vergleich
Filename:
MIME Type:
Creator:
Philipp Hahn
Created:
2012-11-21 16:29:02 CET
Size:
19.80 KB
patch
obsolete
>#!/usr/bin/python2.6 ># pylint: disable-msg=C0301,C0103,C0111,W0704 > >import subprocess >import sys >import os >from optparse import OptionParser >import univention.config_registry as ucr >import tempfile >import shutil >import glob >import datetime >import yaml >import atexit > >configRegistry = ucr.ConfigRegistry() >configRegistry.load() > >sys.path.append("/usr/lib/repo-ng") >import repo_lib as rp # All the package logic >import gnupg >os.umask(0002) # u+rwx,g+rwx,o+rx > ># This file contains the layout of a single errata announcement mail. It is updated through the repo-ng ># source package >mail_template = "/usr/share/repo-ng/templates/errata-template.txt" > ># The key used to sign the errata mail >key_id = "F510AADA" >key_file = "/etc/archive-keys/security-key.txt" >key_id = "A7353B90" # FIXME: PHahn private test key on dimma >key_file = "/home/phahn/univention" > ># This file contains the layout of a single errata HTML page. It is updated through the repo-ng ># source package >html_template = "/usr/share/repo-ng/templates/errata-template.html" > ># This file contains the layout for the overview page linking to all errata HTML page. It is updated through the repo-ng ># source package >overview_template = "/usr/share/repo-ng/templates/overview-template.html" >TYPES = { > 'bugfix': 'Bugfix', > 'sec': 'Security bugfix', > } > ># univention-errata-level needs to be auto-generated, this dict provides a mapping to the respective patch: >ERRATALEVELPATCH = { > '3.1': 'univention-errata-level/3.1-0-0-ucs/1.0.0-1-errata3.1-0/bump-version.patch', > } > ># univention-errata-level needs to be auto-generated, this dict provides a mapping to the respective patch: >ERRATABUILDSCRIPT = { > '3.1': 'b31-scope', > } > > > ># This directory holds the announcement mails, which are about to be sent out once passed through QA >mail_staging_dir = "/var/univention/buildsystem2/errata/staging/" > ># This is the internal version of the errata website. It is mirrored to the public forge site once ># passed through QA >website_dir = "/mnt/omar/vmwares/mirror/ftp/download/errata" > >def remove_file(fn): > try: > os.remove( fn ) > except OSError: > pass > # print 'ERROR: Deleting %s failed' % fn > # print str(e) > > >def generate_release_file(mirror_base, destname, keyfile, keyid): > """ > Create a signed Release file. > > mirror_base : The base directory of the mirror, e.g. /var/univention/buildsystem2/test_mirror/ftp/3.0 > destname : destination directory, e.g. 3.0-0 or errata16 > keyfile : A text file containing the passphrase of the archive key > keyid : The archive, with which the Release file is signed > """ > print 'Generating Release files...' > > for state in ('maintained', 'unmaintained'): > for arch in ( 'all', 'i386', 'amd64' ): > try: > p = os.path.join(mirror_base, state, destname, arch) > os.chdir(p) > except OSError: > print >> sys.stderr, 'Skipping failed cd %s' % (p,) > continue > # Delete old Release file because apt-ftparchive includes it into the new Release file. > remove_file('Release') > f = open('Release.tmp', 'w') > try: > rv = subprocess.call(("apt-ftparchive", > "-o", "APT::FTPArchive::Release::Origin=Univention", > "-o", "APT::FTPArchive::Release::Label=Univention", > "-o", "APT::FTPArchive::Release::Version=%s" % (destname), > "-o", "APT::FTPArchive::Release::Codename=%s/%s" % (destname, arch), > "release", "."), stdout=f) > finally: > f.close() > if rv != 0: > print >> sys.stderr, 'Failed to create %s/Release' % (p,) > remove_file('Release.tmp') > continue > shutil.move('Release.tmp', 'Release') > # Variante mit Debug: rv = subprocess.call(("repo-ng-sign-release-file", "-d", "-p", keyfile, "-k", keyid)) > rv = subprocess.call(("repo-ng-sign-release-file", "-p", keyfile, "-k", keyid)) > if rv != 0: > print >> sys.stderr, 'Failed to sign %s/Release' % (p,) > if os.path.exists('Release.gpg'): > remove_file('Release.gpg') > > >def append_template_overview(): > o = open(overview_data_file, "a") > if errata_31_style: > print >> o, " ".join((errata_id, errata_type, adv['src'], str(adv['version']))) > else: > print >> o, " ".join((errata_id, errata_type, adv['src'])) > o.close() > > >def generate_errata_tgz(mirror_base, errata_id, arch): > """Generate a .tar.gz file containing the updated packages and Packages > files and return the name of the .tar.gz file.""" > tgz_name = tempfile.mktemp('_%s.tar.gz' % (errata_id,)) > tmpdir = tempfile.mkdtemp() > try: > for part in ('maintained', 'unmaintained'): > for arch2 in ('all', arch): > dst_dir = os.path.join(tmpdir, opt.ucsrelease, part, str(errata_id), arch2) > os.makedirs(dst_dir) > src_dir = os.path.join(mirror_base, part, str(errata_id), arch2) > if os.path.exists(src_dir): > subprocess.call(("cp", "-aT", src_dir, dst_dir)) > cmd = ("tar", "cfzC", tgz_name, tmpdir, opt.ucsrelease) > subprocess.call(cmd) > finally: > shutil.rmtree(tmpdir) > > return tgz_name > > >def generate_errata_mail(): > i = open(mail_template, "r") > mail_file = os.path.join(mail_staging_dir, '%s-%s.txt' % (opt.ucsrelease, errata_id)) > > o = open(mail_file, "w") > > print "Generating errata mail in staging directory:", mail_file > > for a in i.readlines(): > a = a.replace("SOURCEPACKAGE", adv['src']) > a = a.replace("UCSVERSION", str(opt.ucsrelease)) > a = a.replace("REFERENCE", adv['ref']) > a = a.replace("ERRID", errata_id) > a = a.replace("FIXED", adv['fix']) > a = a.replace("DESC", adv['desc']) > > if adv['note']: > a = a.replace("NOTE", adv['note']) > else: > a = a.replace("NOTE", "") > > o.write(a) > > o.close() > > gpg = gnupg.GPG(gpgbinary='/usr/bin/gpg') > o = open(mail_file, "rb") > signature = gpg.sign_file(o, clearsign=True, keyid=key_id, passphrase=sec_passphrase) > o.close() > > if signature: > sig_file = open(mail_file + ".asc", 'w') > sig_file.write(str(signature)) > sig_file.close() > else: > print >> sys.stderr, "Failed to generate advisory signature" > sys.exit(1) > > >def send_errata_test_mail(): > """Send test advisory mail.""" > adv_file = os.path.join(mail_staging_dir, '%s-%s.txt.asc' % (opt.ucsrelease, errata_id)) > cmd = ('/usr/bin/errata-mailing', > '-e', str(next_id), > '-v', "Univention Corporate Server", > '-t', adv_file, > '-l', '/var/univention/buildsystem2/errata/mail/test.csv', > '--really') > > print "Sending test advisory mail" > subprocess.call(cmd) > > >def parse_debs_from_changes_files(changes_file): > pkgs = set() > > try: > f = open(changes_file, "r") > except IOError: > print >> sys.stderr, "Failed to open changes file '%s'" % (changes_file,) > sys.exit(1) > > for i in f.readlines(): > l = i.replace("\n", "") > if l.endswith(".deb"): > pkgs.add(l.split(" ")[-1]) > > f.close() > return pkgs > > >def copy_single_source_to_mirror(package, srcdir, targetdir): > src = os.listdir(srcdir) > for j in src: > if j.startswith(package): > if j.endswith(".dsc") or j.endswith(".gz") or j.endswith(".bz2") or j.endswith(".xz"): > if not j.endswith("Sources.gz"): > print "Copying", j, "to source mirror" > shutil.copy(os.path.join(srcdir, j), targetdir) > > >def generate_errata_html(): > """Generate errata web page.""" > > if errata_31_style: > errata_html_file = os.path.join(website_dir, '%s-%s.html' % (opt.ucsrelease, errata_id)) > else: > errata_html_file = os.path.join(website_dir, errata_id + ".html") > > print "Generating errata web page:", errata_html_file > > i = open(html_template, "r") > o = open(errata_html_file, "w") > > errata_header = "UCS " + opt.ucsrelease + " erratum" > > ref_blob = "<tr> \n <td>No external references </td> \n </tr> \n" > > if adv['ref'].count("CVE") > 0: > ref_blob = "" > errata_header = errata_header + " (security)" > for a in adv['ref'].split(","): > ref_blob += '<tr>\n' > ref_blob += '<td><b>CVE ID</b></td>' > ref_blob += '<td><a href="http://cve.mitre.org/cgi-bin/cvename.cgi?name=' + a + '"> ' + a + '</a></td>\n' > ref_blob += '</tr>\n' > else: > ref_blob = "" > errata_header = errata_header + " (bugfix)" > for a in adv['ref'].split(","): > bugnum = a.replace("Bug ", "") > ref_blob += '<tr>\n' > ref_blob += '<td><b>UCS Bug number</b></td>' > ref_blob += '<td><a href="https://forge.univention.org/bugzilla3/show_bug.cgi?id=' + bugnum + '"> UCS bug ' + bugnum + '</a></td>\n' > ref_blob += '</tr>\n' > > currdate = datetime.date.isoformat(datetime.datetime.now()) > > for a in i.readlines(): > a = a.replace("ERRATAHEADER", errata_header) > a = a.replace("ERRATAID", errata_id) > a = a.replace("CURRENTDATE", currdate) > a = a.replace("SRCPKGNAME", adv['src']) > a = a.replace("FIXEDVERSION", adv['fix']) > a = a.replace("DESCRIPTION", adv['desc']) > a = a.replace("REFERENCEBLOB", ref_blob) > a = a.replace("NOTEADVISORY", adv['note'] or "") > o.write(a) > > o.close() > > >def generate_html_overview(): > """Generate errata overview web page.""" > print "Generating errata overview web page:", overview_html_file > errata_header = "Security and bugfix errata for Univention Corporate Server " + opt.ucsrelease > > ref_blob = "" > > data_file = open(overview_data_file, "r") > for line in data_file: > > if errata_31_style: > eid, typ, src, version = line.strip().split(" ", 3) > else: > eid, typ, src = line.strip().split(" ", 2) > > version_blob = str(opt.ucsrelease) > if errata_31_style: > if int(version) == 0: > version_blob = "%s (applicable to all releases)" % (opt.ucsrelease,) > else: > version_blob = "%s (applicable to %s-%s onwards)" % (opt.ucsrelease, opt.ucsrelease, version) > > if errata_31_style: > ref_blob += '<a href="%s-%s.html">%s for UCS %s</a>\n' % (opt.ucsrelease, eid, eid, opt.ucsrelease) > else: > ref_blob += '<a href="%s.html">%s for UCS %s</a>\n' % (eid, eid, opt.ucsrelease) > > ref_blob += '%s for UCS %s (%s)<br/>\n' % (TYPES.get(typ, typ), opt.ucsrelease, src) > data_file.close() > > template_file = open(overview_template, "r") > overview_file = open(overview_html_file, "w") > for a in template_file.readlines(): > a = a.replace("OVERVIEWHEADER", errata_header) > a = a.replace("ERRATABLOB", ref_blob) > overview_file.write(a) > template_file.close() > overview_file.close() > > >def update_errata_package(errata_level): > """Increment version/errata_level in univention-errata-level package and build it.""" > tmpdir = tempfile.mkdtemp('', 'apt-fetch-') > atexit.register(shutil.rmtree, tmpdir, ignore_errors=True) > os.chdir(tmpdir) > > build_password = open('/etc/build.secret').readline().strip('') > svn_dir, svn_file = os.path.split(ERRATALEVELPATCH[opt.ucsrelease]) > svn_path = 'svn+ssh://build@%s/%s' % (configRegistry['repong/patchrepo'], svn_dir) > rp.run_expect('svn co %s svn' % (svn_path,), build_password, 10800) > patchfile = os.path.join('svn', svn_file) > > if os.path.exists(patchfile): > i = open(patchfile, "r") > o = open(patchfile + ".tmp", "w") > PREFIX = "+ucr set version/erratalevel=" > for line in i: > if line.startswith(PREFIX): > line = '%s%d\n' % (PREFIX, errata_level) > o.write(line) > i.close() > o.close() > shutil.move(patchfile + ".tmp", patchfile) > rp.run_expect("svn ci -m 'Package updated by announce-errata (Bug #28893)' %s" % (patchfile,), build_password, 10800) > > # Rebuilding the package > cmd = (ERRATABUILDSCRIPT[opt.ucsrelease], opt.scope, "univention-errata-level") > print ' '.join(cmd) > ret = subprocess.call(cmd) > if ret != 0: > print >> sys.stderr, "Build failed!" > sys.exit(ret) > else: > print >> sys.stderr, "Could not regenerate univention-errata-level package, bailing out" > sys.exit(1) > >tag_svn = False > >p = OptionParser() >p.add_option("-r", action="store", type="string", dest="ucsrelease", > help="The UCS release on which this erratum is based") >p.add_option("-s", action="store", type="string", dest="scope", > help="The scope in which the errata packages are build") >p.add_option("-a", action="store", type="string", dest="advisory", > help="Advisory meta data") >p.add_option("-p", action="store", type="string", dest="package", > help="The package, for which the errata is issued") >p.add_option("-k", "--keyfile", dest="keyfile", action="store", > help="The path to the file containing the archive key") >p.add_option("-K", "--keyid", dest="keyid", action="store", > help="The key-ID of the PGP archive key") >p.add_option("--announce-only", action="store_true", dest="announce_only", default=False, > help="Only generate website / announce mail") >p.add_option("--overview-only", action="store_true", dest="overview_only", > help="Only generate overview website") > >opt, args = p.parse_args() > >if not opt.ucsrelease: > p.error("You need to specify the release for which the erratum should be announced (e.g. -r 3.0)") > >if opt.ucsrelease == "3.0": > errata_31_style = False >else: > errata_31_style = True > > if opt.ucsrelease not in ERRATALEVELPATCH: > print >> sys.sdterr, "No patch mapping has been defined for this release in repo-ng" > sys.exit(1) > > if opt.ucsrelease not in ERRATABUILDSCRIPT: > print >> sys.sdterr, "No build script has been defined for this release in repo-ng" > sys.exit(1) > ># This file contains the basic information on each errata and is used to generate the HTML overview page ># It is created if it not yet exists >overview_data_file = "/var/univention/buildsystem2/errata/status/overview-" + opt.ucsrelease + ".txt" > ># This file contains an overview for all errata for a UCS release >overview_html_file = website_dir + "/overview-" + opt.ucsrelease + ".html" > >if opt.overview_only: > generate_html_overview() > sys.exit(0) > >if not opt.scope: > p.error("You need to specify the scope for which an erratum should be announced (e.g. -s errata3.0)") > >if opt.ucsrelease.count(".") > 1 or opt.ucsrelease.count("-") > 0: > p.error("You need to specify only the major UCS release, e.g. -r 3.0") > >if not opt.advisory: > p.error("You need to specify advisory meta data (e.g. -a emacs.yaml)") > >if not opt.package: > p.error("You need to specify the source package name for which an erratum will be issued (e.g. -p emacs)") > >if not opt.keyfile: > p.error("You need to specifiy a keyfile (e.g. -k /etc/archive-keys/ucs3.0.txt)") > >if not os.path.exists(opt.keyfile): > p.error("Key file not found") > >if not opt.keyid: > p.error("You need to specifiy a key ID (e.g. -K 2CBDA4B0)") > ># This file contains the next errata ID to be used >next_id_file = "/var/univention/buildsystem2/errata/status/next-" + opt.ucsrelease + ".txt" >try: > idfile = open(next_id_file, "r") >except OSError, ex: > print >> sys.stderr, "Error opering %s: %s" % (next_id_file, ex) > sys.exit(1) >next_id = int(idfile.readline()) >idfile.close() > >errata_id = "errata%d" % (next_id,) > >stream = open(opt.advisory, "r") >try: > adv = yaml.load(stream) >finally: > stream.close() > >errata_type = "bugfix" > >if adv['ref'].count("Bug") > 0 and adv['ref'].count("CVE") > 0: > print "Malformed advisory file, you can't mix security and bugfix errata updates" > sys.exit(1) > >if errata_31_style: > if not adv.has_key('version'): > print "Malformed advisory file, no version specified" > sys.exit(1) > else: > errata_base_version = int(adv['version']) > >if adv['ref'].count("CVE") > 0: > errata_type = "sec" > >try: > passfile = open(key_file, "r") >except IOError, ex: > print >> sys.stderr, "Failed to open passphrase file: %s" % (ex,) > sys.exit(1) >try: > sec_passphrase = passfile.read() >finally: > passfile.close() > > >if errata_31_style: > update_errata_package(next_id) > >generate_errata_mail() >send_errata_test_mail() > >generate_errata_html() > >append_template_overview() > >generate_html_overview() > >if opt.announce_only: > print "Only generating announce HTML / mail, exiting" > sys.exit(0) > >mirror_base = "/mnt/omar/vmwares/mirror/ftp/" + opt.ucsrelease > >mirror_base_tarballs = "/mnt/omar/vmwares/mirror/ftp/updates/errata" > >buildsystem_basedir = "/var/univention/buildsystem2/apt/" >apt_base = "/var/univention/buildsystem2/apt/ucs_" + opt.ucsrelease + "-0-" + opt.scope > >archs = ["all", "i386", "amd64"] >apt_server = "apt.univention.de" > >maintained_dir = mirror_base + "/maintained/" >unmaintained_dir = mirror_base + "/unmaintained/" > >maintained_pkgs = [] > >for i in ['i386', 'amd64']: > if opt.ucsrelease == "3.0": # Some packages moved to maintained in 3.0-2, so use a different list > f = open("/var/univention/buildsystem2/cd-contents/ucs_" + str(opt.ucsrelease) + "-2_" + i + "_dvd.txt", "r") > else: > f = open("/var/univention/buildsystem2/cd-contents/ucs_" + str(opt.ucsrelease) + "-0_" + i + "_dvd.txt", "r") > > for j in f.readlines(): > pkgname = j[2:].split("_")[0] # Format looks like this: ./apt-mirror_0.4.8-3.21.201109051552_all.deb > maintained_pkgs.append(pkgname) > >if errata_31_style: > for patch_level in range(errata_base_version, 10): > comp = '%s-%s-errata' % (opt.ucsrelease, patch_level) > errata_dir_maintained = os.path.join(mirror_base, "maintained", "component", comp) > errata_dir_unmaintained = os.path.join(mirror_base, "unmaintained", "component", comp) >else: > errata_dir_maintained = os.path.join(mirror_base, "maintained", errata_id) > errata_dir_unmaintained = os.path.join(mirror_base, "unmaintained", errata_id) > >try: > os.makedirs(errata_dir_maintained) > os.makedirs(errata_dir_unmaintained) >except OSError: > pass > >for i in archs: > try: > os.mkdir(os.path.join(errata_dir_maintained, i)) > os.mkdir(os.path.join(errata_dir_unmaintained, i)) > except OSError: > pass > >try: > os.mkdir(os.path.join(errata_dir_unmaintained, "source")) >except OSError: > pass > ># Parse the changes files for the given source package and copy the files according to their ># maintenance status > >bdir = buildsystem_basedir + "ucs_" + opt.ucsrelease + "-0-" + opt.scope + "/" > >i386_changes_files = glob.glob(bdir + "/source/" + opt.package + "_*changes") >if len(i386_changes_files) == 0: > print "No i386 changes file found, aborting" > sys.exit(1) >if len(i386_changes_files) > 1: > print "More than one changes file found, aborting" > sys.exit(1) > >all_package = False > >amd64_changes_files = glob.glob(bdir + "/amd64/" + opt.package + "_*changes") >if len(amd64_changes_files) == 0: > print "No amd64 changes file found. This is okay if the updated package is an all package" > all_package = True >if len(amd64_changes_files) > 1: > print "More than one changes file found, aborting" > sys.exit(1) > >i386_changes_file = i386_changes_files[0] >for j in parse_debs_from_changes_files(i386_changes_file): > binary_name = j.split("_")[0].lower() > arch = "i386" > if j.endswith("all.deb"): > arch = "all" > > if maintained_pkgs.count(binary_name) > 0: > rp.copy_deb_to_mirror(os.path.join(apt_base, arch, j), True, errata_dir_maintained) > else: > rp.copy_deb_to_mirror(os.path.join(apt_base, arch, j), False, errata_dir_unmaintained) > >if all_package == False: > amd64_changes_file = amd64_changes_files[0] > for j in parse_debs_from_changes_files(amd64_changes_file): > binary_name = j.split("_")[0].lower() > arch = "amd64" > if j.endswith("all.deb"): > arch = "all" > > if maintained_pkgs.count(binary_name) > 0: > rp.copy_deb_to_mirror(os.path.join(apt_base, arch, j), True, errata_dir_maintained) > else: > rp.copy_deb_to_mirror(os.path.join(apt_base, arch, j), False, errata_dir_unmaintained) > >print os.path.join(buildsystem_basedir, "ucs_" + opt.ucsrelease + "-" + opt.scope, "source") >print os.path.join(errata_dir_unmaintained, "source") >copy_single_source_to_mirror(opt.package, os.path.join(bdir, "source"), os.path.join(errata_dir_unmaintained, "source")) > > >if errata_31_style: > comp = '%s-%s-errata' % (opt.ucsrelease, 0) # FIXME: why always 0? > rp.create_packages_files(os.path.join(mirror_base, "maintained", "component"), comp) > rp.create_packages_files(os.path.join(mirror_base, "unmaintained", "component"), comp) > rp.create_sources_files(os.path.join(mirror_base, "unmaintained", "component"), comp) > generate_release_file(mirror_base, "component/" + comp, opt.keyfile, opt.keyid) >else: > rp.create_packages_files(os.path.join(mirror_base, "maintained"), errata_id) > rp.create_packages_files(os.path.join(mirror_base, "unmaintained"), errata_id) > rp.create_sources_files(os.path.join(mirror_base, "unmaintained"), errata_id) > generate_release_file(mirror_base, errata_id, opt.keyfile, opt.keyid) > > > >if not errata_31_style: > for arch in ('amd64', 'i386'): > tgz = generate_errata_tgz(mirror_base, errata_id, arch) > name = os.path.join(mirror_base_tarballs, "ucs-errata-%s-%s_%s.tar.gz" % (opt.ucsrelease, errata_id, arch)) > shutil.move(tgz, name) >else: > pass # TODO > ># Update next ID last in case of failures >idfile = open(next_id_file, "w") >idfile.write(str(next_id + 1) >idfile.close()
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Raw
Actions:
View
Attachments on
bug 28893
: 4826