Attachment 8996 Details for Bug 44928

[patch] nt-diagnostic-logcheck.patch

nt-diagnostic-logcheck.patch (text/plain), 12.00 KB, created by Lukas Oyen on 2017-07-04 12:30 CEST

(hide)

Description:

Filename:

MIME Type:

Creator: Lukas Oyen

Created: 2017-07-04 12:30 CEST

Size: 12.00 KB

patch

obsolete

>From 796ab39e6e5c5538f14dbe45179da104f1c1465b Mon Sep 17 00:00:00 2001
>From: Lukas Oyen <oyen@univention.de>
>Date: Thu, 29 Jun 2017 16:55:05 +0200
>Subject: [PATCH 1/2] Bug #XXX: diagnostic: new check `logcheck.py`
>
>---
> .../umc/python/diagnostic/plugins/logcheck.py      | 258 +++++++++++++++++++++
> 1 file changed, 258 insertions(+)
> create mode 100755 management/univention-management-console-module-diagnostic/umc/python/diagnostic/plugins/logcheck.py
>
>diff --git a/management/univention-management-console-module-diagnostic/umc/python/diagnostic/plugins/logcheck.py b/management/univention-management-console-module-diagnostic/umc/python/diagnostic/plugins/logcheck.py
>new file mode 100755
>index 0000000..9a510b9
>--- /dev/null
>+++ b/management/univention-management-console-module-diagnostic/umc/python/diagnostic/plugins/logcheck.py
>@@ -0,0 +1,258 @@
>+#!/usr/bin/python2.7
>+# coding: utf-8
>+#
>+# Univention Management Console module:
>+#  System Diagnosis UMC module
>+#
>+# Copyright 2017 Univention GmbH
>+#
>+# http://www.univention.de/
>+#
>+# All rights reserved.
>+#
>+# The source code of this program is made available
>+# under the terms of the GNU Affero General Public License version 3
>+# (GNU AGPL V3) as published by the Free Software Foundation.
>+#
>+# Binary versions of this program provided by Univention to you as
>+# well as other copyrighted, protected or trademarked materials like
>+# Logos, graphics, fonts, specific documentations and configurations,
>+# cryptographic keys etc. are subject to a license agreement between
>+# you and Univention and not subject to the GNU AGPL V3.
>+#
>+# In the case you use this program under the terms of the GNU AGPL V3,
>+# the program is provided in the hope that it will be useful,
>+# but WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU Affero General Public License for more details.
>+#
>+# You should have received a copy of the GNU Affero General Public
>+# License with the Debian GNU/Linux or Univention distribution in file
>+# /usr/share/common-licenses/AGPL-3; if not, see
>+# <http://www.gnu.org/licenses/>.
>+
>+import re
>+import mmap
>+import errno
>+import socket
>+import itertools as it
>+
>+from univention.management.console.modules.diagnostic import Warning
>+
>+from univention.lib.i18n import Translation
>+_ = Translation('univention-management-console-module-diagnostic').translate
>+
>+title = _('Check logfiles for errors')
>+description = _('No errors found in logfiles.')
>+
>+
>+class Logcheck(object):
>+	# The `error_regex` should be a `re.compile()`ed regex matching error
>+	# markers in log messages. Log messages are either single log lines, or
>+	# multi line log entries, if `split_messages()` is overridden (as in
>+	# `MultiLineLogcheck`). This is used as `error_regex.search()` to find log
>+	# messages that contain any error marker.
>+	# example: `ERROR_REGEX = re.compile('(ERROR)|(CRITICAL)')`
>+	ERROR_REGEX = None
>+
>+	# The `start_regex` should be a `re.compile()`ed regex matching the start
>+	# of a service. This should include the start of the line and not only the
>+	# start marker. This is used as `start_regex.finditer()` to find the last
>+	# service start in a logfile and seek to it.
>+	# example: `START_REGEX = re.compile('^.*Starting service xxx')`
>+	START_REGEX = None
>+
>+	def __init__(self, filename, error_regex=None, start_regex=None):
>+		self.filename = filename
>+		self.error_regex = error_regex or self.ERROR_REGEX
>+		self.start_regex = start_regex or self.START_REGEX
>+
>+		self.fob = open(self.filename)
>+		# We use a `mmap`, as we can't use `re` on files and reading all the
>+		# content of a file might by too slow.
>+		self.mmap = mmap.mmap(self.fob.fileno(), 0, access=mmap.ACCESS_READ)
>+
>+		if self.start_regex is not None:
>+			self._seek_to_last_start()
>+
>+	@classmethod
>+	def with_file(cls, filename, **kwargs):
>+		def wrapper():
>+			return cls(filename, **kwargs)
>+		return wrapper
>+
>+	@classmethod
>+	def with_error_phrases(cls, filename, *phrases, **kwargs):
>+		regex = re.compile('|'.join('({})'.format(phrase) for phrase in phrases))
>+		return cls.with_file(filename, error_regex=regex, **kwargs)
>+
>+	def __enter__(self):
>+		return self
>+
>+	def __exit__(self):
>+		self.close()
>+
>+	def _seek_to_last_start(self):
>+		# find the last `match` object by exhausting the iterator
>+		match = None
>+		for match in self.start_regex.finditer(self.mmap):
>+			pass
>+		start = match.start() if match is not None else 0
>+		self.mmap.seek(start)
>+
>+	def close(self):
>+		if self.mmap is not None:
>+			self.mmap.close()
>+		if self.fob is not None:
>+			self.fob.close()
>+
>+	def split_messages(self):
>+		'''
>+		Yield all lines like `file.splitlines()`.
>+		Override this for multiline log entries.
>+		'''
>+		while True:
>+			line = self.mmap.readline()
>+			if line == '':
>+				break
>+			yield line.rstrip('\n')
>+
>+	def errors(self):
>+		for message in self.split_messages():
>+			search = self.error_regex.search(message)
>+			if search is not None and search.span() != (0, 0):
>+				yield message
>+
>+
>+class MultiLineLogcheck(Logcheck):
>+	# The `message_start_regex` should be a `re.compile()`ed regex matching the
>+	# start of a multiline log message. This is used to group together multiple
>+	# log lines and used as `message_start_regex.search()` to find if a line is
>+	# a message start.
>+	MESSAGE_START_REGEX = None
>+
>+	def __init__(self, filename, message_start_regex=None, **kwargs):
>+		super(MultiLineLogcheck, self).__init__(filename, **kwargs)
>+		self.message_start_regex = message_start_regex or self.MESSAGE_START_REGEX
>+
>+	def split_messages(self):
>+		chunk = list()
>+		for line in super(MultiLineLogcheck, self).split_messages():
>+			if self.message_start_regex.search(line) is not None:
>+				if chunk:
>+					yield '\n'.join(chunk)
>+					chunk = list()
>+			chunk.append(line)
>+		if chunk:
>+			yield '\n'.join(chunk)
>+
>+
>+class SyslogLogCheck(Logcheck):
>+	SERVICE_NAME = None
>+
>+	def __init__(self, filename, service_name=None, **kwargs):
>+		super(SyslogLogCheck, self).__init__(filename, **kwargs)
>+		self.service_name = service_name or self.SERVICE_NAME
>+
>+	def split_messages(self):
>+		if self.service_name is not None:
>+			regex_tmpl = '^.*({})|(unassigned-hostname) {}\[\d+\]:'
>+			service_regex = re.compile(regex_tmpl.format(socket.gethostname(), self.service_name))
>+			for line in super(SyslogLogCheck, self).split_messages():
>+				if service_regex.search(line) is not None:
>+					yield line
>+		else:
>+			for line in super(SyslogLogCheck, self).split_messages():
>+				yield line
>+
>+
>+class SambaLogcheck(MultiLineLogcheck):
>+	# comma separated logging header (level is not reliable):
>+	# <date>, <level> [, pid=<pid>] [, <effective user>, <real user>]
>+	# see samba source: lib/util/debug.c `dbghdrclass()`
>+	MESSAGE_START_REGEX = re.compile((
>+		'^\['
>+		'\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}.\d{6}'
>+		', ( \d)|(\d{2}).'
>+		'(, pid=\d+)?'
>+		'(, effective$\d+, \d+$), real$\d+, \d+$?'
>+		'\]'
>+	))
>+
>+
>+class UniventionDebugLogcheck(Logcheck):
>+	ERROR_REGEX = re.compile('$\s*ERROR\s*$')
>+	START_REGEX = re.compile('^.*DEBUG_INIT.*$', re.MULTILINE)
>+
>+	@classmethod
>+	def with_levels(cls, filename, *levels, **kwargs):
>+		phrases = ('$\s*{}\s*$'.format(l) for l in levels)
>+		return cls.with_error_phrases(filename, *phrases, **kwargs)
>+
>+
>+class MultiLineUniventionDebugLogcheck(MultiLineLogcheck, UniventionDebugLogcheck):
>+	MESSAGE_START_REGEX = re.compile('^\d{2}.\d{2}.\d{4} \d{2}:\d{2}:\d{2},\d{3}')
>+
>+
>+class UniventionUpdaterLogcheck(UniventionDebugLogcheck):
>+	ERROR_REGEX = re.compile('(CRITICAL)|(error code)|(Hash Sum mismatch)')
>+	START_REGEX = re.compile('^Starting univention-upgrade. Current UCS version is', re.MULTILINE)
>+
>+
>+LOGFILES = {
>+	'univention/listener':
>+		UniventionDebugLogcheck.with_file('/var/log/univention/listener.log'),
>+	'univention/notifier':
>+		UniventionDebugLogcheck.with_file('/var/log/univention/notifier.log'),
>+	'univention/updater':
>+		UniventionUpdaterLogcheck.with_file('/var/log/univention/updater.log'),
>+	'univention/s4connector':
>+		MultiLineUniventionDebugLogcheck.with_levels('/var/log/univention/connector-s4.log',
>+			'ERROR', 'WARNING'),
>+	'syslog/named':
>+		SyslogLogCheck.with_file('/var/log/syslog', service_name='named',
>+			error_regex=re.compile("update '.*' denied")),
>+	'samba/samba':
>+		SambaLogcheck.with_error_phrases('/var/log/samba/log.samba', 'BACKTRACE'),
>+	'samba/smbd':
>+		SambaLogcheck.with_error_phrases('/var/log/samba/log.smbd', 'BACKTRACE'),
>+}
>+
>+
>+class ErrorFound(Exception):
>+	def __init__(self, name, filename, error_message):
>+		super(ErrorFound, self).__init__(name, filename, error_message)
>+		self.name = name
>+		self.filename = filename
>+		self.error_message = error_message
>+
>+	def __str__(self):
>+		return self.error_message
>+
>+
>+def find_log_errors():
>+	for (name, check_wrapper) in LOGFILES.iteritems():
>+		try:
>+			check = check_wrapper()
>+		except IOError as error:
>+			if error.errno != errno.ENOENT:
>+				raise
>+		else:
>+			for error in check.errors():
>+				yield ErrorFound(name, check.filename, error)
>+
>+
>+def run():
>+	errors = list(find_log_errors())
>+	if errors:
>+		ed = [_('Some errors in logfiles found.')]
>+		grouped = it.groupby(errors, lambda e: (e.name, e.filename))
>+		for ((name, filename), group) in grouped:
>+			ed.append(_('\nIn logfile {log!r}:').format(log=filename))
>+			ed.extend(str(e) for e in group)
>+		raise Warning(description='\n'.join(ed))
>+
>+
>+if __name__ == '__main__':
>+	from univention.management.console.modules.diagnostic import main
>+	main()
>-- 
>2.7.4
>
>
>From 57bcf291c40639411ea0b3ac834eb8bd79e7d919 Mon Sep 17 00:00:00 2001
>From: Lukas Oyen <oyen@univention.de>
>Date: Tue, 4 Jul 2017 12:15:29 +0200
>Subject: [PATCH 2/2] Bug #XXX: diagnostic: new check `logcheck.py` (po)
>
>---
> .../umc/python/diagnostic/de.po                    | 24 ++++++++++++++++++++--
> 1 file changed, 22 insertions(+), 2 deletions(-)
>
>diff --git a/management/univention-management-console-module-diagnostic/umc/python/diagnostic/de.po b/management/univention-management-console-module-diagnostic/umc/python/diagnostic/de.po
>index affad86..e58743a 100644
>--- a/management/univention-management-console-module-diagnostic/umc/python/diagnostic/de.po
>+++ b/management/univention-management-console-module-diagnostic/umc/python/diagnostic/de.po
>@@ -2,8 +2,8 @@
> msgid ""
> msgstr ""
> "Project-Id-Version: univention-management-console-module-diagnostic\n"
>-"Report-Msgid-Bugs-To: packages@univention.de\n"
>-"POT-Creation-Date: 2016-01-14 12:19+0100\n"
>+"Report-Msgid-Bugs-To: \n"
>+"POT-Creation-Date: 2017-07-04 12:14+0200\n"
> "PO-Revision-Date: \n"
> "Last-Translator: Univention GmbH <packages@univention.de>\n"
> "Language-Team: Univention GmbH <packages@univention.de>\n"
>@@ -12,6 +12,14 @@ msgstr ""
> "Content-Type: text/plain; charset=UTF-8\n"
> "Content-Transfer-Encoding: 8bit\n"
> 
>+#: umc/python/diagnostic/plugins/logcheck.py:251
>+msgid ""
>+"\n"
>+"In logfile {log!r}:"
>+msgstr ""
>+"\n"
>+"In der Logdatei {log!r}:"
>+
> #: umc/python/diagnostic/plugins/nameserver.py:15
> #, python-format
> msgid "%d of the configured nameservers are not responding to DNS queries."
>@@ -27,6 +35,10 @@ msgstr ""
> msgid "Adjust to suggested limits"
> msgstr "An vorgeschlagene Limits anpassen"
> 
>+#: umc/python/diagnostic/plugins/logcheck.py:45
>+msgid "Check logfiles for errors"
>+msgstr "ÃberprÃ¼fe Logdateien auf Fehler"
>+
> #: umc/python/diagnostic/plugins/gateway.py:11
> msgid "Gateway is not reachable"
> msgstr "Gateway ist nicht erreichbar"
>@@ -97,6 +109,10 @@ msgstr ""
> msgid "Nameserver(s) are not responsive"
> msgstr "Nameserver sind nicht ansprechbar"
> 
>+#: umc/python/diagnostic/plugins/logcheck.py:46
>+msgid "No errors found in logfiles."
>+msgstr "Keine Fehler in Logdateien gefunden."
>+
> #: umc/python/diagnostic/plugins/package_status.py:11
> msgid "Package status corrupt"
> msgstr "Paketstatus korrupt"
>@@ -137,6 +153,10 @@ msgstr "SSH-Verbindung zu anderem UCS Server fehlgeschlagen!"
> msgid "Security limits exceeded"
> msgstr "Sicherheitslimits Ã¼berschritten"
> 
>+#: umc/python/diagnostic/plugins/logcheck.py:248
>+msgid "Some errors in logfiles found."
>+msgstr "Es wurden Fehler in Logdateien gefunden."
>+
> #: umc/python/diagnostic/__init__.py:262
> msgid "Test again"
> msgstr "Erneut testen"
>-- 
>2.7.4
>

Actions: View | Diff

Attachments on bug 44928: 8996