Attachment #7832 for bug #41223




#!/usr/bin/make -f
# -*- coding: utf-8 -*-
#
# Copyright 2016 Univention GmbH
#
# http://www.univention.de/
#
# All rights reserved.
#
# The source code of this program is made available
# under the terms of the GNU Affero General Public License version 3
# (GNU AGPL V3) as published by the Free Software Foundation.
#
# Binary versions of this program provided by Univention to you as
# well as other copyrighted, protected or trademarked materials like
# Logos, graphics, fonts, specific documentations and configurations,
# cryptographic keys etc. are subject to a license agreement between
# you and Univention and not subject to the GNU AGPL V3.
#
# In the case you use this program under the terms of the GNU AGPL V3,
# the program is provided in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License with the Debian GNU/Linux or Univention distribution in file
# /usr/share/common-licenses/AGPL-3; if not, see
# <http://www.gnu.org/licenses/>.

# This Makefile is copied to translation source packages generated by
# univention-ucs-translation-build-package. It defines the receipts to build
# binary translation targets(mo files). The include 'all_targets.mk' is
# generated by the same script and contains all actual targets.

include all_targets.mk

%.mo:
	mkdir -p $(@D)
	msgfmt --check --output-file="$@" "$<"

%.json:
	dh-umc-po2json "$<"  # creates .json file in the same directory as the .po file
	mkdir -p $(@D)
	mv "$(patsubst %.po,%.json,$<)" "$@"

build:

install: $(ALL_TARGETS)

(-)a/packaging/univention-ucs-translation-template/debian/univention-ucs-translation-template.install (-1 / +3 lines)

Lines 1-4	Link Here

translationhelper.py usr/share/pyshared/univention

translationhelper.py usr/share/pyshared/univention

specialcases.json usr/share/univention-ucs-translation-template

specialcases.json usr/share/univention-ucs-translation-template

univention-ucs-translation-build-package.py usr/bin

base_makefile usr/share/univention-ucs-translation-template

univention-ucs-translation-merge usr/bin

univention-ucs-translation-build-package usr/bin

univention-ucs-translation-fakemessage usr/bin

univention-ucs-translation-fakemessage usr/bin




	'0001-6-7'
]

MAKEFILE_HEADER = """#!/usr/bin/make -f

%.mo:
	mkdir -p $(@D)
	msgfmt --check --output-file="$@" "$<"

%.json:
	#python -c "import univention.dh_umc as dhumc; dhumc.create_json_file('$<')"
	dh-umc-po2json "$<"  # creates .json file in the same directory as the .po file
	mkdir -p $(@D)
	mv "$(patsubst %.po,%.json,$<)" "$@"
	#install -D $($<:.po=.json) "$@"

"""

MAKEFILE_END = """build:

install: $(ALL_TARGETS)
"""


class UMCModuleTranslation(dh_umc.UMC_Module):
	def __init__(self, attrs, target_language):

		try:
			module = UMCModuleTranslation._get_core_module_from_source_package(module_in_source_tree, target_language)
		except AttributeError as e:
			print("%s core module load failed" % str(e))
			# TODO: Module not loaded at all --> exception?
		else:
			print("Successfully loaded as core module: {}".format(module_in_source_tree.get('abs_path_to_src_pkg')))

		attrs['relative_path_src_pkg'] = module.get('relative_path_src_pkg')
		return UMCModuleTranslation(attrs, target_language)


# TODO: actually.. (module, output_dir)
def update_package_translation_files(module, output_dir):
	print("Creating directories and PO files for {module_name} in translation source package".format(**module))
	start_dir = os.getcwd()

					try:
						dh_umc.create_po_file(new_po_file_abs_path, module['module_name'], src_files)
					except dh_umc.Error as exc:
						print(str(exc))

			# build python po files
			_create_po_files(module.python_po_files, module.python_files)

			try:
				dh_umc.module_xml2po(module, po_file_full_path, lang)
			except dh_umc.Error as exc:
				print(str(exc))

	except OSError as exc:
		print(traceback.format_exc())
		print("error in update_package_translation_files: %s" % (exc,))
	finally:
		os.chdir(start_dir)
	return abs_path_translated_src_pkg


	for scase in special_cases:
		_append_to_target_lists(scase.get('mo_destination'), '{lang}/{po_subdir}/{lang}.po'.format(lang=target_language, **scase))
	with open(os.path.join(new_package_dir, 'all_targets.mk'), 'w') as fd:
		fd.write("# This file is auto-generated by univention-ucs-translation-build-package and should not be edited!\n\n")
		fd.write('ALL_TARGETS = {}\n\n'.format(' \\\n\t'.join(mo_targets_list)))
		fd.write('\n'.join(target_prerequisite))
		fd.write('\n')
		fd.writelines(MAKEFILE_END)


# special case e.g. univention-management-modules-frontend: translation files are built with a makefile

	# TODO: Checks whether a special case is valid should be done on special case parsing
	path_src_pkg = os.path.join(source_dir, special_case.get('package_dir'))
	if not os.path.isdir(path_src_pkg):
		# TODO: Exception
		return

	new_po_path = os.path.join(output_dir, special_case.get('po_subdir'))

	if not matches:
		# TODO: Exception
		print('Error: specialcase for {} didn\'t match any files.'.format(special_case.get('package_dir')))
	# FIXME: create_po should handle path itself?
	cwd = os.getcwd()
	os.chdir(path_src_pkg)
	try:
		matches = [os.path.relpath(match, start=os.getcwd()) for match in matches]
		dh_umc.create_po_file(new_po_path, special_case.get('package_name'), matches)
	except dh_umc.Error as exc:
		repr(exc)
	except TypeError as exc:
		repr(exc)
	os.chdir(cwd)


def find_base_translation_modules(startdir, source_dir, module_basefile_name):
	print('looking in %s' % source_dir)
	print('looking for files matching %s' % module_basefile_name)
	os.chdir(source_dir)
	matches = []
	for root, dirnames, filenames in os.walk('.'):


	regex = re.compile(".*/(.*)/debian/.*%s$" % re.escape(module_basefile_name))
	for match in matches:
		print(match)
		packagenameresult = regex.search(match)
		if packagenameresult:
			packagename = packagenameresult.group(1)

			modulename = os.path.basename(match.replace(module_basefile_name, ''))
			if modulename in MODULE_BLACKLIST:
				print("Ignoring module %s: Module is blacklisted\n" % modulename)
				continue

			package_dir = os.path.dirname(os.path.dirname(match))
			print("Found package: %s" % package_dir)
			module = {}
			module['module_name'] = modulename
			module['binary_package_name'] = packagename

			module['relative_path_src_pkg'] = os.path.relpath(package_dir)
			base_translation_modules.append(module)
		else:
			print("could not obtain packagename from directory %s" % match)

	os.chdir(startdir)
	return base_translation_modules

def create_new_package(new_package_dir, target_language, target_locale, language_name, startdir):
	new_package_dir_debian = os.path.join(new_package_dir, 'debian')
	if not os.path.exists(new_package_dir_debian):
		print("creating directory: %s" % new_package_dir_debian)
		os.makedirs(new_package_dir_debian)

	translation_package_name = "univention-ucs-translation-%s" % target_language

usr/share/locale/%(lang)s/LC_MESSAGES
""" % language_dict)

	shutil.copyfile('/usr/share/univention-ucs-translation-template/base_makefile', os.path.join(new_package_dir, 'Makefile'))
	# Move source files and installed .mo files to new package dir
	if os.path.exists(os.path.join(new_package_dir, 'usr')):
		shutil.rmtree(os.path.join(new_package_dir, 'usr'))
	#shutil.copytree(os.path.join(startdir, 'usr'), os.path.join(new_package_dir, 'usr'))

		shutil.rmtree(os.path.join(new_package_dir, target_language))
	shutil.copytree(os.path.join(startdir, target_language), os.path.join(new_package_dir, target_language))
	shutil.rmtree(os.path.join(startdir, target_language))





#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2013-2014 Univention GmbH
#
# http://www.univention.de/
#
# All rights reserved.
#
# The source code of this program is made available
# under the terms of the GNU Affero General Public License version 3
# (GNU AGPL V3) as published by the Free Software Foundation.
#
# Binary versions of this program provided by Univention to you as
# well as other copyrighted, protected or trademarked materials like
# Logos, graphics, fonts, specific documentations and configurations,
# cryptographic keys etc. are subject to a license agreement between
# you and Univention and not subject to the GNU AGPL V3.
#
# In the case you use this program under the terms of the GNU AGPL V3,
# the program is provided in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License with the Debian GNU/Linux or Univention distribution in file
# /usr/share/common-licenses/AGPL-3; if not, see
# <http://www.gnu.org/licenses/>.

from optparse import OptionParser

import os
import json
import univention.dh_umc as dh_umc
import univention.translationhelper as tlh


SPECIALCASES_PATH = "/usr/share/univention-ucs-translation-template/specialcases.json"


if __name__ == '__main__':
	usage = '''%prog [options] -s source_dir -c language_code -l locale -n language_name
e.g.: -s /path/to/ucs-repository/ -c de -l de_DE.UTF-8:UTF-8 -n Deutsch'''
	parser = OptionParser(usage=usage)
	parser.add_option('-s', '--source', action='store', dest='source_dir', help='UCS source dir from which translation files are gathered, e.g. an UCS svn base dir')
	parser.add_option('-c', '--languagecode', action='store', dest='target_language', help='Target language code (e.g. de)')
	parser.add_option('-b', '--basefiles', action='store', dest='basefiles', default='.umc-modules', help='xml file basename (default: .umc-modules)')
	parser.add_option('-l', '--locale', action='store', dest='target_locale', help='Target locale (e.g. de_DE.UTF-8:UTF-8)')
	parser.add_option('-n', '--languagename', action='store', dest='target_name', help='Language name that is shown in the UMC (e.g. Deutsch)')

	(options, args) = parser.parse_args()
	help_message = 'Use --help to show additional help.'

	if not options.source_dir:
		parser.error('Missing argument -s. %s' % help_message)

	if not options.target_language:
		parser.error('Missing argument -c. %s' % help_message)

	if not options.target_locale:
		parser.error('Missing argument -l. %s' % help_message)

	if not options.target_name:
		parser.error('Missing argument -n. %s' % help_message)

	options.source_dir = os.path.abspath(options.source_dir)
	# find all module files and move them to a language specific directory
	startdir = os.getcwd()
	base_translation_modules = tlh.find_base_translation_modules(startdir, options.source_dir, options.basefiles)
	dh_umc.LANGUAGES = (options.target_language, )
	all_modules = list()
	output_dir = os.path.join(os.getcwd(), options.target_language)
	for module_attrs in base_translation_modules:
		module = tlh.UMCModuleTranslation.from_source_package(module_attrs, options.target_language)
		all_modules.append(module)
		abs_path_translated_src_pkg = tlh.update_package_translation_files(module, output_dir)

	# special cases, e.g. univention-management-console-frontend
	special_cases = []
	if os.path.exists(SPECIALCASES_PATH):
		with open(SPECIALCASES_PATH, 'rb') as fd:
			special_cases = json.load(fd)
	else:
		print("Error: Could not find file {}. Several files will not be handled.".format(SPECIALCASES_PATH))

	for s_case in special_cases:
		tlh.translate_special_case(s_case, options.source_dir, options.target_language, output_dir)

	# create new package
	new_package_dir = os.path.join(startdir, 'univention-ucs-translation-%s' % options.target_language)
	tlh.create_new_package(new_package_dir, options.target_language, options.target_locale, options.target_name, startdir)
	tlh.write_makefile(all_modules, special_cases, new_package_dir, options.target_language)




#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2013-2014 Univention GmbH
#
# http://www.univention.de/
#
# All rights reserved.
#
# The source code of this program is made available
# under the terms of the GNU Affero General Public License version 3
# (GNU AGPL V3) as published by the Free Software Foundation.
#
# Binary versions of this program provided by Univention to you as
# well as other copyrighted, protected or trademarked materials like
# Logos, graphics, fonts, specific documentations and configurations,
# cryptographic keys etc. are subject to a license agreement between
# you and Univention and not subject to the GNU AGPL V3.
#
# In the case you use this program under the terms of the GNU AGPL V3,
# the program is provided in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public
# License with the Debian GNU/Linux or Univention distribution in file
# /usr/share/common-licenses/AGPL-3; if not, see
# <http://www.gnu.org/licenses/>.

from optparse import OptionParser

import os
import json
import univention.dh_umc as dh_umc
import univention.translationhelper as tlh


SPECIALCASES_PATH = "/usr/share/univention-ucs-translation-template/specialcases.json"


if __name__ == '__main__':
	usage = '''%prog [options] -s source_dir -c language_code -l locale -n language_name
e.g.: -s /path/to/ucs-repository/ -c de -l de_DE.UTF-8:UTF-8 -n Deutsch'''
	parser = OptionParser(usage=usage)
	parser.add_option('-s', '--source', action='store', dest='source_dir', help='UCS source dir from which translation files are gathered, e.g. an UCS svn base dir')
	parser.add_option('-c', '--languagecode', action='store', dest='target_language', help='Target language code (e.g. de)')
	parser.add_option('-b', '--basefiles', action='store', dest='basefiles', default='.umc-modules', help='xml file basename (default: .umc-modules)')
	parser.add_option('-l', '--locale', action='store', dest='target_locale', help='Target locale (e.g. de_DE.UTF-8:UTF-8)')
	parser.add_option('-n', '--languagename', action='store', dest='target_name', help='Language name that is shown in the UMC (e.g. Deutsch)')

	(options, args) = parser.parse_args()
	help_message = 'Use --help to show additional help.'

	if not options.source_dir:
		parser.error('Missing argument -s. %s' % help_message)

	if not options.target_language:
		parser.error('Missing argument -c. %s' % help_message)

	if not options.target_locale:
		parser.error('Missing argument -l. %s' % help_message)

	if not options.target_name:
		parser.error('Missing argument -n. %s' % help_message)

	options.source_dir = os.path.abspath(options.source_dir)
	# find all module files and move them to a language specific directory
	startdir = os.getcwd()
	base_translation_modules = tlh.find_base_translation_modules(startdir, options.source_dir, options.basefiles)
	dh_umc.LANGUAGES = (options.target_language, )
	all_modules = list()
	output_dir = os.path.join(os.getcwd(), options.target_language)
	for module_attrs in base_translation_modules:
		module = tlh.UMCModuleTranslation.from_source_package(module_attrs, options.target_language)
		all_modules.append(module)
		abs_path_translated_src_pkg = tlh.update_package_translation_files(module, output_dir)

	# special cases, e.g. univention-management-console-frontend
	special_cases = []
	if os.path.exists(SPECIALCASES_PATH):
		with open(SPECIALCASES_PATH, 'rb') as fd:
			special_cases = json.load(fd)
	else:
		print "Error: Could not find file %s. Several files will not be handled." % SPECIALCASES_PATH

	for s_case in special_cases:
		tlh.translate_special_case(s_case, options.source_dir, options.target_language, output_dir)

	# create new package
	new_package_dir = os.path.join(startdir, 'univention-ucs-translation-%s' % options.target_language)
	tlh.create_new_package(new_package_dir, options.target_language, options.target_locale, options.target_name, startdir)
	tlh.write_makefile(all_modules, special_cases, new_package_dir, options.target_language)




Usage: univention-ucs-translation-fakemessage [directory]

This script walks over all po files in a given directory and its subfolders.
It fills all msgid strings with the same content as the corresponding msgid,
prepeding it with the character sequence '!TR!'.
It is useful for visual quick testing of translation coverage and po file
generation.
"""


def main(po_dir):
	for _dir, dns, fns in os.walk(os.path.abspath(po_dir)):
		for po_file in fnmatch.filter(fns, '*.po'):
			po = polib.pofile(os.path.join(_dir, po_file))
			for entry in po:
				if entry.msgid == '':

(-)a/packaging/univention-ucs-translation-template/univention-ucs-translation-merge (-11 / +37 lines)

Lines 29-40	Link Here

# <http://www.gnu.org/licenses/>.

# <http://www.gnu.org/licenses/>.

import argparse

import argparse

import fnmatch

import fnmatch

import polib

import os

import os

import sys

import sys

import shutil

import univention.translationhelper as tlh

import univention.translationhelper as tlh

import univention.dh_umc as dh_umc

import univention.dh_umc as dh_umc

import json

import json

import subprocess

from pdb import set_trace as dbg

SPECIALCASES_PATH = "/usr/share/univention-ucs-translation-template/specialcases.json"

SPECIALCASES_PATH = "/usr/share/univention-ucs-translation-template/specialcases.json"

Lines 50-55 def _get_po_files_from_dir(directory, return_relative_path=False):	Link Here

	return po_files

	return po_files

def _msgmerge(source, destination):

	try:

		subprocess.check_call(['msgmerge', '--update', '--sort-output', destination, source])

	except TypeError:

		print('Error: arguments must be names of existing files. Exiting.')

		sys.exit(1)

	except subprocess.CalledProcessError:

		print('Error: PO file merge failed. Exiting.')

		sys.exit(1)

def merge_po_file_trees(source_tree, target_tree):

def merge_po_file_trees(source_tree, target_tree):

	upstream_pos = _get_po_files_from_dir(source_tree, return_relative_path=True)

	upstream_pos = _get_po_files_from_dir(source_tree, return_relative_path=True)

	translated_pos = _get_po_files_from_dir(target_tree, return_relative_path=True)

	translated_pos = _get_po_files_from_dir(target_tree, return_relative_path=True)

Lines 62-77 def merge_po_file_trees(source_tree, target_tree):	Link Here

	# add new

	# add new

	for new_po in new_pos:

	for new_po in new_pos:

		new_po_path = os.path.join(target_tree, new_po)

		new_po_path = os.path.join(target_tree, new_po)

		os.makedirs(os.path.dirname(new_po_path))

		try:

			os.makedirs(os.path.dirname(new_po_path))

		except OSError as exc:

			if not exc.errno == 17:

				raise

			pass

		os.rename(os.path.join(source_tree, new_po), new_po_path)

		os.rename(os.path.join(source_tree, new_po), new_po_path)

		upstream_pos.remove(new_po)

	# merge upstream changes...

	# merge upstream changes...

	for dir, dirs, files in os.walk(target_tree):

	for po_file in upstream_pos:

		for po_file in fnmatch.filter([os.path.join(dir, fn) for fn in files], '*.po'):

		_msgmerge(os.path.join(source_tree, po_file), os.path.join(target_tree, po_file))

			translated = polib.pofile(po_file)

	return pos_delete_upstream, new_pos

			upstream = polib.pofile(po_file.replace(target_tree, source_tree))

			translated.merge(upstream)

			translated.save()

if __name__ == '__main__':

if __name__ == '__main__':

	parser = argparse.ArgumentParser(description="Merge upstream changes in translation files to existing translation source package.")

	parser = argparse.ArgumentParser(description="Merge upstream changes in translation files to existing translation source package.")

Lines 91-98 if __name__ == '__main__':	Link Here

	dh_umc.LANGUAGES = (args.lang_code, )

106

	dh_umc.LANGUAGES = (args.lang_code, )

	modules_upstream = tlh.find_base_translation_modules(os.getcwd(), args.upstream, '.umc-modules')

107

	modules_upstream = tlh.find_base_translation_modules(os.getcwd(), args.upstream, '.umc-modules')

	output_dir = os.path.join(args.translation, '{}_merge'.format(args.lang_code))

108

	output_dir = os.path.join(args.translation, '{}_merge'.format(args.lang_code))

109

	modules = list()

	for module_attrs in modules_upstream:

110

	for module_attrs in modules_upstream:

		module = tlh.UMCModuleTranslation.from_source_package(module_attrs, args.lang_code)

111

		module = tlh.UMCModuleTranslation.from_source_package(module_attrs, args.lang_code)

112

		modules.append(module)

		tlh.update_package_translation_files(module, output_dir)

113

		tlh.update_package_translation_files(module, output_dir)

	# special cases, e.g. univention-management-console-frontend

114

	# special cases, e.g. univention-management-console-frontend

	special_cases = []

115

	special_cases = []

Lines 100-108 if __name__ == '__main__':	Link Here

100

		with open(SPECIALCASES_PATH, 'rb') as fd:

117

		with open(SPECIALCASES_PATH, 'rb') as fd:

101

			special_cases = json.load(fd)

118

			special_cases = json.load(fd)

102

	else:

119

	else:

103

		print "Error: Could not find file %s. Several files will not be handled." % SPECIALCASES_PATH

120

		print("Error: Could not find file {}.\nSeveral files will not be handled.".format(SPECIALCASES_PATH))

104

121

105

	for scase in special_cases:

122

	for scase in special_cases:

106

		tlh.translate_special_case(scase, args.upstream, args.lang_code, output_dir)

123

		tlh.translate_special_case(scase, args.upstream, args.lang_code, output_dir)

107

124

108

	merge_po_file_trees(output_dir, os.path.join(args.translation, args.lang_code))

125

	removed, new = merge_po_file_trees(output_dir, os.path.join(args.translation, args.lang_code))

126

	print('Merge summary: ')

127

	if removed:

128

		print('Following files have been removed:')

129

		print('\n'.join(removed))

130

	if new:

131

		print('Following files habe been added:')

132

		print('\n'.join(new))

133

	shutil.rmtree(output_dir)

134

	tlh.write_makefile(modules, special_cases, args.translation, args.lang_code)

Return to bug 41223