147 lines
5.0 KiB
Python
147 lines
5.0 KiB
Python
#!/usr/bin/python
|
|
# encoding: utf-8
|
|
|
|
"""
|
|
Sõltuvuste paigladamiseks:
|
|
|
|
apt install -y python-jinja2 python-lxml python-pygal python-geoip python-numpy python-matplotlib
|
|
dnf install -y python-jinja2 python-lxml python-pygal python2-GeoIP python2-numpy python2-matplotlib
|
|
|
|
Käivitamiseks
|
|
|
|
(cat /path/to/access.log; zcat /path/to/access.log.1.gz) | python main.py
|
|
(cat /path/to/access.log; zcat /path/to/access.log.[12].gz) | python main.py
|
|
(cat /path/to/access.log; zcat /path/to/access.log.[1-5].gz) | python main.py
|
|
(cat /path/to/access.log; zcat /path/to/access.log.*.gz) | python main.py
|
|
|
|
"""
|
|
|
|
import GeoIP
|
|
import re
|
|
import sys
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
from log_parsers import ApacheLogParser
|
|
|
|
then = datetime.now()
|
|
hits_per_remote_addr = Counter() # IP-d kust tuldi
|
|
hits_per_path = Counter() # URL-id mida külastati
|
|
hits_per_user_agent = Counter() # User agent mida kasutati külastamisel
|
|
hits_per_country = Counter() # Riigid kust päringud tulid
|
|
hits_per_date = Counter() # Kuupäevad mil logikirjeid oli
|
|
bytes_per_date = Counter()
|
|
|
|
gi = GeoIP.open("/usr/share/GeoIP/GeoIP.dat", GeoIP.GEOIP_MEMORY_CACHE)
|
|
|
|
print "Loen standardsisendist..."
|
|
for timestamp, log_entry, stack_trace, byte_count, line_count, event_count in ApacheLogParser(sys.stdin):
|
|
country_code = gi.country_code_by_addr(log_entry.get("remote_addr"))
|
|
if not country_code:
|
|
# Jäta vahele sisevõrgu aadressid, localhost jms mis ei lahendu riigiks
|
|
continue
|
|
if int(log_entry.get("status")) < 400: # 2xx ja 3xx arvestamiseks
|
|
hits_per_path[log_entry.get("path")] += 1
|
|
hits_per_remote_addr[log_entry.get("remote_addr")] += 1
|
|
hits_per_user_agent[log_entry.get("user_agent")] += 1
|
|
hits_per_country[country_code] += 1
|
|
hits_per_date[timestamp.date()] += 1
|
|
bytes_per_date[timestamp.date()] += int(log_entry.get("size"))
|
|
|
|
# Leia kõige esimene kuupäev ning viimane kuupäev millal sündmused esinesid
|
|
first_date, last_date = min(hits_per_date.keys()), max(hits_per_date.keys())
|
|
|
|
############################
|
|
### Värvi riigid kaardil ###
|
|
############################
|
|
|
|
import requests
|
|
from lxml import etree
|
|
from numpy import interp
|
|
|
|
# Laadi alla kaart wikimedia veebist
|
|
print "Laadin alla kaarti..."
|
|
buf = requests.get("https://upload.wikimedia.org/wikipedia/commons/0/03/BlankMap-World6.svg").content
|
|
|
|
# Parsi XML puu
|
|
map_document = etree.fromstring(buf)
|
|
for country, count in hits_per_country.items():
|
|
if not country:
|
|
# Mõni IP ei pruukinud laheneda riigikoodiks (sisevõrk jms)
|
|
continue
|
|
hue = interp(count, [0, max(hits_per_country.values())], [180, 0])
|
|
for element in map_document.xpath("//*[@id='%s']" % country.lower()):
|
|
element.set("style", "fill:hsl(%.2f, 60%%, 60%%)" % hue)
|
|
for subelement in element:
|
|
subelement.attrib.pop("class", "")
|
|
|
|
with open("top.svg", "wb") as fh:
|
|
fh.write(etree.tostring(map_document))
|
|
print "Kaart salvestatud faili top.svg"
|
|
|
|
|
|
####################################################
|
|
### Koosta päringute arvu graafik päevade lõikes ###
|
|
####################################################
|
|
|
|
# matplotlibi serveris kasutamiseks tuleb välja vahetada vaikimisi joonsitamise backend
|
|
# https://matplotlib.org/tutorials/introductory/usage.html#what-is-a-backend
|
|
import matplotlib
|
|
matplotlib.use("Agg")
|
|
|
|
import matplotlib.pyplot as plt
|
|
fig = plt.figure(figsize=(10, 7))
|
|
|
|
# Lisa joonise sisse kaks graafikut
|
|
sub = fig.add_subplot(2, 1, 1)
|
|
sub2 = fig.add_subplot(2, 1, 2)
|
|
|
|
sub.set_xlabel(u"Päringute arv")
|
|
sub2.set_xlabel(u"Liikluse maht baitides")
|
|
|
|
|
|
# Ploti andmepunktid
|
|
sub.barh(hits_per_date.keys(), hits_per_date.values())
|
|
sub2.barh(bytes_per_date.keys(), bytes_per_date.values())
|
|
|
|
# Salvesta faili
|
|
fig.savefig("bar.svg", format="svg")
|
|
fig.savefig("bar.png")
|
|
|
|
|
|
#######################################
|
|
### Koosta veebilehitsejate graafik ###
|
|
#######################################
|
|
|
|
import pygal
|
|
line_chart = pygal.Pie(
|
|
truncate_legend=50, # legend kuni 50 karakterit
|
|
width=1000, height=300, # graafiku laius/kõrgus pikslites
|
|
style=pygal.style.Style(background='transparent')) # eemalda taustavärv
|
|
line_chart.config(style_name = 'LightStyle', fill=None)
|
|
line_chart.title = 'Veebilehitsejate osakaal'
|
|
for user_agent, count in hits_per_user_agent.most_common(7):
|
|
line_chart.add(user_agent, count) # lisa graafikule
|
|
hits_per_user_agent.pop(user_agent) # eemalda counteri objektist
|
|
line_chart.add("Muud veebilehitsejad", sum(hits_per_user_agent.values()))
|
|
|
|
user_agent_chart = line_chart.render(is_unicode=True, disable_xml_declaration=True)
|
|
|
|
|
|
|
|
from jinja2 import Template
|
|
import codecs
|
|
|
|
# Loe jinja mall UTF-8 tekstifailist
|
|
with codecs.open("template.html", "rb", encoding="utf-8") as fh:
|
|
template = Template(fh.read())
|
|
# Süsti malli sisse kõik kohalikud muutujad (first_date, last_date, hits, urls jne)
|
|
buf = template.render(locals())
|
|
|
|
# Salvesta täidetud leht UTF-8 kodeeringus
|
|
with codecs.open("raport.html", "wb", encoding="utf-8") as fh:
|
|
fh.write(buf)
|
|
|
|
print "HTML kujul raport savlestatud faili raport.html"
|
|
|
|
|