logide-parsimine/raport/log_parsers.py

# encoding: utf-8
import re
import sys
from datetime import datetime

class GenericParser(object):
    def __init__(self, fh, errors_only=False): # siia võid anda sys.stdin, gzip.open, open vms file handle tüüpi obj
        self.fh = fh
        self.errors_only = errors_only

    def __iter__(self):
        multiline_message = ""
        log_entry = None
        byte_count = 0
        line_count = 0
        event_count = 0
        for line in self.fh:
            # Loenda logikirjete arv ja maht
            byte_count += len(line)
            line_count += 1

            # Normaliseeri reavahetused
            line = line.replace("\r\n", "\n")

            # Jäta vahele tühjad read
            if not line.strip():
                continue

            # Püüa regulaaravaldise järgi rida tükkideks võtta
            m = re.match(self.RE_LOG_ENTRY, line)

            # Kui rida klappis regexiga
            if m:
                # Väljasta eelmine kokku kleebitud logikirje
                if log_entry:
                    # Kui vaja filtreerida, välista logikirjed mis ei ole veateatega seotud
                    if not self.errors_only or self.is_serious(log_entry):
                        event_count += 1

                        # yield teeb funktsioonist generaatori/iteraatori
                        # https://pythontips.com/2013/09/29/the-python-yield-keyword-explained/
                        yield datetime.strptime(log_entry.get("timestamp"), self.TIMESTAMP_FORMAT), \
                            log_entry, multiline_message, byte_count, line_count, event_count


                # Alusta järgmise sõnumi kokku kleepimist
                multiline_message = m.group("message")
                log_entry = m.groupdict()

            elif line.startswith("\t") or line.startswith("Caused by") or line.startswith("org."):
                multiline_message += line

            else:
                sys.stderr.write("Ei suutnud parsida rida:" + line)

class JavaLogParser(GenericParser):
    TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S.%f"
    RE_LOG_ENTRY = "(?P<timestamp>.+?) +(?P<severity>[A-Z]+) 1 --- \[(?P<thread>.+)\](?P<class>.+?) *: (?P<message>.+)"

    def is_serious(self, log_entry):
        return log_entry.get("severity") == "ERROR"

class ApacheLogParser(GenericParser):
    RE_LOG_ENTRY = "(?P<remote_addr>.+?) - (?P<username>.+?) \[(?P<timestamp>.+?) \+\d\d\d\d\] \"(?P<message>(?P<verb>[A-Z]+) (?P<path>.+) HTTP/1.[01])\" (?P<status>\d+) (?P<size>\d+) \"(?P<referrer>.+?)\" \"(?P<user_agent>.+?)\""
    TIMESTAMP_FORMAT = "%d/%b/%Y:%H:%M:%S"

    def is_serious(self, log_entry):
        return int(log_entry.get("status")) >= 400