Attachment #8069 for bug #41882

View | Details | Raw Unified | Return to bug 41882
Collapse All | Expand All




# /usr/share/common-licenses/AGPL-3; if not, see
# <http://www.gnu.org/licenses/>.

import sys
import datetime

from ucsschool.importer.factory import Factory

from ucsschool.importer.configuration import Configuration
from ucsschool.importer.utils.logging import get_logger
from ucsschool.lib.models.utils import stopped_notifier
from memory_profiler import profile, LogFile


class MassImport(object):

		self.result_exporter = self.factory.make_result_exporter()
		self.password_exporter = self.factory.make_password_exporter()
		self.errors = list()
		sys.stdout = LogFile(self.logger.name, reportIncrementFlag=False)

	def mass_import(self):
		with stopped_notifier():

	def import_routers(self):
		pass

	@profile(precision=3)
	def import_users(self):
		self.logger.info("------ Importing users... ------")
		user_import = self.factory.make_user_importer(self.dry_run)

		user_import.create_and_modify_users(imported_users)
		self.errors.extend(user_import.errors)
		user_import.log_stats()
		if False and self.config["output"]["new_user_passwords"]:  # TODO
			nup = datetime.datetime.now().strftime(self.config["output"]["new_user_passwords"])
			self.logger.info("------ Writing new users passwords to %s... ------", nup)
			self.password_exporter.dump(user_import, nup)




from ucsschool.importer.configuration import Configuration
from ucsschool.importer.utils.logging import get_logger
from ucsschool.importer.utils.ldap_connection import get_admin_connection
from memory_profiler import profile, LogFile


class UserImport(object):

		self.connection, self.position = get_admin_connection()
		self.factory = Factory()
		self.reader = self.factory.make_reader()
		sys.stdout = LogFile(self.logger.name, reportIncrementFlag=False)

	@profile(precision=3)
	def read_input(self):
		"""
		Read users from input data.

		self.logger.info("------ Read %d users from input data. ------", len(self.imported_users))
		return self.imported_users

	@profile(precision=3)
	def create_and_modify_users(self, imported_users=None):
		"""
		Create and modify users.

		:return tuple: (self.errors, self.added_users, self.modified_users)
		"""
		self.logger.info("------ Creating / modifying users... ------")
		usernum = 0
		total = len(imported_users)
		while imported_users:
			imported_user = imported_users.pop(0)
			usernum += 1
			if imported_user.action == "D":
				continue
			try:
				self.logger.debug("Creating / modifying user %d/%d %s...", usernum, total, imported_user)
				user = self.determine_add_modify_action(imported_user)
				cls_name = user.__class__.__name__


						user.record_uid, exc), validation_error=exc, import_user=user), sys.exc_info()[2]

				if success:
					self.logger.info("Success %s %d/%d %s (source_uid:%s record_uid: %s).", action_str.lower(), usernum, 						total, user, user.source_uid, user.record_uid)
					store.append(user.to_dict_all())
					store.append(user)
				else:
					raise err("Error {} {}/{} {} (source_uid:{} record_uid: {}), does probably {}exist.".format(
						action_str.lower(), usernum, len(imported_users), user, user.source_uid, user.record_uid,
						"not " if user.action == "M" else "already "), entry=user.entry_count, import_user=user)

			except (CreationError, ModificationError) as exc:
				self.logger.error("Entry #%d: %s",  exc.entry, exc)  # traceback useless
				self._add_error(exc)

			user.action = "A"
		return user

	@profile(precision=3)
	def detect_users_to_delete(self):
		"""
		Find difference between source database and UCS user database.

		"""
		self.logger.info("------ Detecting which users to delete... ------")
		users_to_delete = list()
		a_user = self.factory.make_import_user([])

		if self.config["no_delete"]:
			self.logger.info("------ Looking only for users with action='D' (no_delete=%r) ------",

			for user in self.imported_users:
				if user.action == "D":
					try:
						users_to_delete.append((user.source_uid, user.record_uid))
						ldap_user.update(user)  # need user.input_data for hooks
						users_to_delete.append(ldap_user)
					except noObject:
						msg = "User to delete not found in LDAP: {}.".format(user)
						self.logger.error(msg)

		ucs_ldap_users = self.connection.search(filter_s, attr=attr)
		ucs_user_ids = set([(lu[1]["ucsschoolSourceUID"][0], lu[1]["ucsschoolRecordUID"][0]) for lu in ucs_ldap_users])

		users_to_delete = list(ucs_user_ids - imported_user_ids)
		for ucs_id_not_in_import in (ucs_user_ids - imported_user_ids):
			try:
				ldap_user = a_user.get_by_import_id(self.connection, ucs_id_not_in_import[0], ucs_id_not_in_import[1])
				ldap_user.action = "D"  # mark for logging/csv-output purposes
				users_to_delete.append(ldap_user)
			except noObject as exc:
				self.logger.error("Cannot delete non existing user with source_uid=%r, record_uid=%r: %s",
					ucs_id_not_in_import[0], ucs_id_not_in_import[1], exc)

		self.logger.debug("users_to_delete=%r", users_to_delete)
		return users_to_delete

	@profile(precision=3)
	def delete_users(self, users=None):
		"""
		Delete users.

		:return: tuple: (self.errors, self.deleted_users)
		"""
		self.logger.info("------ Deleting %d users... ------", len(users))
		a_user = self.factory.make_import_user([])
		for num, ucs_id_not_in_import in enumerate(users):
			try:
				user = a_user.get_by_import_id(self.connection, ucs_id_not_in_import[0], ucs_id_not_in_import[1])
				user.action = "D"  # mark for logging/csv-output purposes
			except noObject as exc:
				self.logger.error("Cannot delete non existing user with source_uid=%r, record_uid=%r: %s",
								  ucs_id_not_in_import[0], ucs_id_not_in_import[1], exc)
				continue
			try:
				success = self.do_delete(user)
				if success:
					self.logger.info("Success deleting %d/%d %r (source_uid:%s record_uid: %s).", num, len(users),
						user.name, user.source_uid, user.record_uid)
				else:
					raise DeletionError("Error deleting user '{}' (source_uid:{} record_uid: {}), has probably already "
						"been deleted.".format(user.name, user.source_uid, user.record_uid), entry=user.entry_count,
						import_user=user)
				self.deleted_users[user.__class__.__name__].append(user.to_dict_all())
			except UcsSchoolImportError as exc:
				self.logger.exception("Error in entry #%d: %s",  exc.entry, exc)
				self._add_error(exc)

			raise UnknownDeleteSetting("Don't know what to do with user_deletion=%r and expiration=%r.".format(
				self.config["user_deletion"]["delete"], self.config["user_deletion"]["expiration"]),
				entry=user.entry_count, import_user=user)
		user.invalidate_all_caches()
		return success

	def log_stats(self):

		for cls_name in sorted(cls_names):
			self.logger.info("Created %s: %d", cls_name, len(self.added_users.get(cls_name, [])))
			for i in range(0, len(self.added_users[cls_name]), columns):
				self.logger.info("  %s", [iu["name"] for iu in self.added_users[cls_name][i:i+columns]])
			self.logger.info("Modified %s: %d", cls_name, len(self.modified_users.get(cls_name, [])))
			for i in range(0, len(self.modified_users[cls_name]), columns):
				self.logger.info("  %s", [iu["name"] for iu in self.modified_users[cls_name][i:i+columns]])
			self.logger.info("Deleted %s: %d", cls_name, len(self.deleted_users.get(cls_name, [])))
			for i in range(0, len(self.deleted_users[cls_name]), columns):
				self.logger.info("  %s", [iu["name"] for iu in self.deleted_users[cls_name][i:i+columns]])
		self.logger.info("Errors: %d", len(self.errors))
		if self.errors:
			self.logger.info("Entry #: Error description")

(-)modules/ucsschool/importer/models/import_user.py (-14 / +47 lines)

Lines 80-85	Link Here

	reader = None

	reader = None

	logger = None

	logger = None

	_pyhook_cache = None

	_pyhook_cache = None

	_additional_props = ("action", "entry_count", "udm_properties", "input_data", "old_user", "in_hook")

	prop = uadmin_property("_replace")

	def __init__(self, name=None, school=None, **kwargs):

	def __init__(self, name=None, school=None, **kwargs):

		self.action = None            # "A", "D" or "M"

		self.action = None            # "A", "D" or "M"

Lines 88-100	Link Here

		self.input_data = list()      # raw input data created by SomeReader.read()

		self.input_data = list()      # raw input data created by SomeReader.read()

		self.old_user = None          # user in LDAP, when modifying

		self.old_user = None          # user in LDAP, when modifying

		self.in_hook = False          # if a hook is currently running

		self.in_hook = False          # if a hook is currently running

		for attr in self._additional_props:

			try:

				val = kwargs.pop(attr)

				setattr(self, attr, val)

			except KeyError:

				pass

100

		if not self.factory:

101

		if not self.factory:

			self.factory = Factory()

102

			self.__class__.factory = Factory()

			self.ucr = self.factory.make_ucr()

103

			self.__class__.ucr = self.factory.make_ucr()

			self.config = Configuration()

104

			self.__class__.config = Configuration()

			self.reader = self.factory.make_reader()

105

			self.__class__.reader = self.factory.make_reader()

			self.logger = get_logger()

106

			self.__class__.logger = get_logger()

			self.username_max_length = 20 - len(self.ucr.get("ucsschool/ldap/default/userprefix/exam", "exam-"))

107

			self.__class__.username_max_length = 20 - len(self.ucr.get("ucsschool/ldap/default/userprefix/exam", "exam-"))

		self._lo = None

108

		self._lo = None

		self._userexpiry = None

109

		self._userexpiry = None

100

		super(ImportUser, self).__init__(name, school, **kwargs)

110

		super(ImportUser, self).__init__(name, school, **kwargs)

Lines 186-191	Link Here

186

"""

196

"""

187

		self._userexpiry = expiry

197

		self._userexpiry = expiry

188

198

199

	@classmethod

200

	def from_dict(cls, a_dict):

201

		if "Staff" in a_dict["type_name"]:

202

			if "Teacher" in a_dict["type_name"]:

203

				kls = ImportTeachersAndStaff

204

			else:

205

				kls = ImportStaff

206

		elif "Student" in a_dict["type_name"]:

207

			kls = ImportStudent

208

		elif "Teacher" in a_dict["type_name"]:

209

			kls = ImportTeacher

210

		else:

211

			kls = cls

212

		a_dict.pop("$dn$", None)

213

		a_dict.pop("type", None)

214

		a_dict.pop("type_name", None)

215

		return kls(**a_dict)

216

189

	def _alter_udm_obj(self, udm_obj):

217

	def _alter_udm_obj(self, udm_obj):

190

		super(ImportUser, self)._alter_udm_obj(udm_obj)

218

		super(ImportUser, self)._alter_udm_obj(udm_obj)

191

		if self._userexpiry is not None:

219

		if self._userexpiry is not None:

Lines 479-485	Link Here

479

			raise FormatError("No username was created from scheme '{}'.".format(

507

			raise FormatError("No username was created from scheme '{}'.".format(

480

				self.username_scheme), self.username_scheme, self.to_dict())

508

				self.username_scheme), self.username_scheme, self.to_dict())

481

		if not self.username_handler:

509

		if not self.username_handler:

482

			self.username_handler = self.factory.make_username_handler(self.username_max_length)

510

			self.__class__.username_handler = self.factory.make_username_handler(self.username_max_length)

483

		self.name = self.username_handler.format_username(self.name)

511

		self.name = self.username_handler.format_username(self.name)

484

512

485

	def modify(self, lo, validate=True, move_if_necessary=None):

513

	def modify(self, lo, validate=True, move_if_necessary=None):

Lines 507-514	Link Here

507

		self._lo = lo

535

		self._lo = lo

508

		return super(ImportUser, self).move(lo, udm_obj, force)

536

		return super(ImportUser, self).move(lo, udm_obj, force)

509

537

510

	@staticmethod

538

	@classmethod

511

	def normalize(s):

539

	def normalize(cls, s):

512

"""

540

"""

513

		Normalize string (german umlauts etc)

541

		Normalize string (german umlauts etc)

514

542

Lines 516-523	Link Here

516

		:return: str: normalized s

544

		:return: str: normalized s

517

"""

545

"""

518

		if isinstance(s, basestring):

546

		if isinstance(s, basestring):

519

			prop = uadmin_property("_replace")

547

			s = cls.prop._replace("<:umlauts>{}".format(s), {})

520

			s = prop._replace("<:umlauts>{}".format(s), {})

521

		return s

548

		return s

522

549

523

	def normalize_udm_properties(self):

550

	def normalize_udm_properties(self):

Lines 665-676	Link Here

665

			all_fields = self.reader.get_data_mapping(self.input_data)

692

			all_fields = self.reader.get_data_mapping(self.input_data)

666

		else:

693

		else:

667

			all_fields = dict()

694

			all_fields = dict()

668

		all_fields.update(self.to_dict().copy())

695

		all_fields.update(self.to_dict())

669

		all_fields.update(self.udm_properties)

696

		all_fields.update(self.udm_properties)

670

		all_fields.update(kwargs)

697

		all_fields.update(kwargs)

671

698

672

		prop = uadmin_property("_replace")

699

		res = self.prop._replace(scheme, all_fields)

673

		res = prop._replace(scheme, all_fields)

674

		if not res:

700

		if not res:

675

			self.logger.warn("Created empty '{prop_name}' from scheme '{scheme}' and input data {data}. ".format(

701

			self.logger.warn("Created empty '{prop_name}' from scheme '{scheme}' and input data {data}. ".format(

676

				prop_name=prop_name, scheme=scheme, data=all_fields))

702

				prop_name=prop_name, scheme=scheme, data=all_fields))

Lines 755-760	Link Here

755

			raise UnknownProperty("UDM properties could not be set. Unknown property: '{}'".format(exc),

781

			raise UnknownProperty("UDM properties could not be set. Unknown property: '{}'".format(exc),

756

				entry=self.entry_count, import_user = self)

782

				entry=self.entry_count, import_user = self)

757

783

784

	def to_dict_all(self):

785

		res = self.to_dict()

786

		for attr in self._additional_props:

787

			res[attr] = getattr(self, attr)

788

		self.logger.debug("res=%r", res)

789

		return res

790

758

	def update(self, other):

791

	def update(self, other):

759

"""

792

"""

760

		Copy attributes of other ImportUser into this one.

793

		Copy attributes of other ImportUser into this one.

(-)modules/ucsschool/importer/writer/user_import_csv_result_exporter.py (-1 / +4 lines)

Lines 72-78	Link Here

		for users in [user_import.added_users, user_import.modified_users, user_import.deleted_users]:

		for users in [user_import.added_users, user_import.modified_users, user_import.deleted_users]:

			tmp = list()

			tmp = list()

			map(tmp.extend, [u for u in users.values() if u])

			map(tmp.extend, [u for u in users.values() if u])

			li.extend(sorted(tmp, key=lambda x: x.entry_count))

			li.extend(tmp)

		return li

		return li

	def get_writer(self):

	def get_writer(self):

Lines 95-100	Link Here

			user = obj

			user = obj

		elif isinstance(obj, UcsSchoolImportError):

		elif isinstance(obj, UcsSchoolImportError):

			user = obj.import_user

			user = obj.import_user

		elif isinstance(obj, dict):

			user = self.factory.make_import_user([]).from_dict(obj)

100

			obj = user

		else:

101

		else:

			raise TypeError("Expected ImportUser or UcsSchoolImportError, got {}. Repr: {}".format(type(obj), repr(obj)))

102

			raise TypeError("Expected ImportUser or UcsSchoolImportError, got {}. Repr: {}".format(type(obj), repr(obj)))

100

		if not user:

103

		if not user:

Return to bug 41882