Apache logiparsija parandused

This commit is contained in:
Lauri Võsandi 2018-06-16 00:55:57 +03:00
parent 0649680459
commit ae9c681c6d
3 changed files with 26 additions and 6 deletions

View File

@ -4,8 +4,9 @@ import sys
from datetime import datetime
class GenericParser(object):
def __init__(self, fh): # siia võid anda sys.stdin, gzip.open, open vms file handle tüüpi obj
def __init__(self, fh, errors_only=False): # siia võid anda sys.stdin, gzip.open, open vms file handle tüüpi obj
self.fh = fh
self.errors_only = errors_only
def __iter__(self):
multiline_message = ""
@ -21,7 +22,9 @@ class GenericParser(object):
continue
m = re.match(self.RE_LOG_ENTRY, line)
if m:
if log_entry and self.is_serious(log_entry):
if log_entry:
if self.errors_only and not self.is_serious(log_entry):
continue
stack_trace = "\n".join(multiline_message.split("\n")[1:])
event_count += 1
row = \

View File

@ -4,8 +4,16 @@
"""
Sõltuvuste paigladamiseks:
apt install -y python-jinja2 python-lxml python-pygal
dnf install -y python-jinja2 python-lxml python-pygal
apt install -y python-jinja2 python-lxml python-pygal python-geoip python-numpy python-matplotlib
dnf install -y python-jinja2 python-lxml python-pygal python2-GeoIP python2-numpy python2-matplotlib
Käivitamiseks
(cat /path/to/access.log; zcat /path/to/access.log.1.gz) | python main.py
(cat /path/to/access.log; zcat /path/to/access.log.[12].gz) | python main.py
(cat /path/to/access.log; zcat /path/to/access.log.[1-5].gz) | python main.py
(cat /path/to/access.log; zcat /path/to/access.log.*.gz) | python main.py
"""
import GeoIP
@ -28,7 +36,11 @@ gi = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_MEMORY_CACHE)
print "Loen standardsisendist..."
for timestamp, log_entry, stack_trace, byte_count, line_count, event_count in ApacheLogParser(sys.stdin):
country_code = gi.country_code_by_addr(log_entry.get("remote_addr"))
hits_per_path[log_entry.get("path")] += 1
if not country_code:
# Jäta vahele sisevõrgu aadressid, localhost jms mis ei lahendu riigiks
continue
if int(log_entry.get("status")) < 400: # 2xx ja 3xx arvestamiseks
hits_per_path[log_entry.get("path")] += 1
hits_per_remote_addr[log_entry.get("remote_addr")] += 1
hits_per_user_agent[log_entry.get("user_agent")] += 1
hits_per_country[country_code] += 1
@ -71,6 +83,11 @@ print "Kaart salvestatud faili top.svg"
### Koosta päringute arvu graafik päevade lõikes ###
####################################################
# matplotlibi serveris kasutamiseks tuleb välja vahetada vaikimisi joonsitamise backend
# https://matplotlib.org/tutorials/introductory/usage.html#what-is-a-backend
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(10, 7))

View File

@ -9,7 +9,7 @@
<script type="text/javascript" src="http://kozea.github.com/pygal.js/latest/pygal-tooltips.min.js"></script>
<style>
svg, img {
max-width: 80%;
max-width: 100%; /* joonise laius 100% lehe laiusest */
}
</style>
</head>