619 lines
24 KiB
Python
619 lines
24 KiB
Python
# python
|
|
from os import listdir, path, stat
|
|
from re import match
|
|
from datetime import date, datetime, timedelta
|
|
from calendar import monthrange
|
|
from csv import reader
|
|
from statistics import mean, median
|
|
from yaml import load as yaml_load
|
|
from requests import get as requests_get
|
|
# django
|
|
from django.shortcuts import render, get_object_or_404, redirect
|
|
from django.http import HttpResponse, HttpResponseNotFound
|
|
from django.conf import settings
|
|
from django.core.exceptions import ObjectDoesNotExist
|
|
from django.core.cache import cache
|
|
from django.db import IntegrityError
|
|
from django.contrib.auth.decorators import login_required
|
|
# project
|
|
from .models import Os, Group, Server, ServerStatus, PackageStatus, Team, Document_Servers, Document_Packages
|
|
from .forms import DocumentForm
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## INDEX
|
|
## Home page
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def index(request):
|
|
return render(request, 'index.html', {})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## SERVER LIST
|
|
## Show servers, their status, number of updates...
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def server_list(request, year=None, month=None, day=None, group=None, team=None):
|
|
# TODO: use date.today?
|
|
now = datetime.now()
|
|
if not year or not month or not day:
|
|
year = now.year
|
|
month = now.month
|
|
day = now.day
|
|
year = int(year)
|
|
month = int(month)
|
|
day = int(day)
|
|
|
|
# check if this is a day
|
|
try:
|
|
current_date = date(year, month, day)
|
|
except ValueError:
|
|
return HttpResponseNotFound('Page not found')
|
|
|
|
# retrieve the date of the fist results before and after current date
|
|
previous_result_date = None
|
|
previous_result = ServerStatus.objects.filter(date__lt=current_date).order_by('date').last()
|
|
if previous_result is not None:
|
|
previous_result_date = previous_result.date
|
|
next_result_date = None
|
|
next_result = ServerStatus.objects.filter(date__gt=current_date).order_by('date').first()
|
|
if next_result is not None:
|
|
next_result_date = next_result.date
|
|
|
|
results_date = current_date
|
|
if group:
|
|
group = get_object_or_404(Group, name=group)
|
|
status_list = ServerStatus.objects.filter(date=current_date, server__group=group).order_by('server__hostname')
|
|
if previous_result and not status_list:
|
|
status_list = ServerStatus.objects.filter(date=previous_result_date, server__group=group).order_by('server__hostname')
|
|
results_date = previous_result_date
|
|
elif team:
|
|
team = get_object_or_404(Team, color=team)
|
|
status_list = ServerStatus.objects.filter(date=current_date, server__team=team).order_by('server__hostname')
|
|
if previous_result and not status_list:
|
|
status_list = ServerStatus.objects.filter(date=previous_result_date, server__team=team).order_by('server__hostname')
|
|
results_date = previous_result_date
|
|
else:
|
|
# status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname')
|
|
status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname').select_related('server', 'server__group', 'server__os')
|
|
if previous_result and not status_list:
|
|
status_list = ServerStatus.objects.filter(date=previous_result_date).order_by('server__hostname').select_related('server', 'server__group', 'server__os')
|
|
results_date = previous_result_date
|
|
|
|
if not status_list:
|
|
return render(request, 'generic.html', {
|
|
'content': '<p>No results found.</p>',
|
|
})
|
|
|
|
return render(request, 'server-list.html',
|
|
{
|
|
'group': group,
|
|
'status_list': status_list,
|
|
'results_date': results_date,
|
|
'previous_result_date': previous_result_date,
|
|
'next_result_date': next_result_date,
|
|
})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## PACKAGES LIST
|
|
## Show package list
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def packages_list(request):
|
|
packages_list_all = [p.package_name for p in PackageStatus.objects.all()]
|
|
packages_list = []
|
|
|
|
for p in set(packages_list_all):
|
|
packages_list.append([p, packages_list_all.count(p)])
|
|
|
|
return render(request, 'packages-list.html', { 'packages_list': packages_list, })
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## SHOW PACKAGE
|
|
## Show servers and versions of a package
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def packages(request, package=None , hostname=None):
|
|
# what do we want?
|
|
if package is not None:
|
|
# show all servers that have this package
|
|
packages = PackageStatus.objects.filter(package_name=package)
|
|
else:
|
|
if hostname is not None:
|
|
# show package for one host
|
|
packages = PackageStatus.objects.filter(server__hostname=hostname)
|
|
else:
|
|
# show all packages for all hosts
|
|
packages = PackageStatus.objects.all()
|
|
|
|
return render(request, 'packages.html', { 'packages': packages, })
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## OS STATISTICS
|
|
## Show percentage for each distrib / version, and a nice chart
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def os_statistics(request):
|
|
# get last date for which we have stats
|
|
last_date = ServerStatus.objects.all().order_by('date').last().date
|
|
current_st_list = ServerStatus.objects.filter(date=last_date)
|
|
nb_servers = len(current_st_list)
|
|
os_list = Os.objects.all()
|
|
os_stat = []
|
|
js_data = ""
|
|
|
|
if last_date is not None:
|
|
# count how many server we have for each distribution
|
|
for os in os_list:
|
|
nb = len(current_st_list.filter(server__os=os))
|
|
|
|
if nb > 0:
|
|
os_stat.append([str(os), nb, nb * 100 / nb_servers])
|
|
# for the chart
|
|
js_data = js_data + """
|
|
{
|
|
label: "%s",
|
|
value: %d
|
|
},""" % (str(os), nb)
|
|
|
|
return render(request, 'os-statistics.html', {
|
|
'js_data': js_data,
|
|
'os_stat': os_stat,
|
|
'last_date': last_date,
|
|
})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## HISTORY
|
|
## Show graphs for a month: updates or uptime or os-eol
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def history(request, obj, year=False, month=False):
|
|
if not year:
|
|
year = datetime.now().year
|
|
if not month:
|
|
month = datetime.now().month
|
|
|
|
if int(month) == int(datetime.now().month):
|
|
current_month = True
|
|
else:
|
|
current_month = False
|
|
|
|
year = int(year)
|
|
month = int(month)
|
|
|
|
pc_js_data = ""
|
|
mean_js_data = ""
|
|
|
|
if current_month:
|
|
num_days = 30
|
|
days = [datetime.now() - timedelta(days=n) for n in range(num_days, -1, -1)]
|
|
else:
|
|
num_days = monthrange(year,month)[1]
|
|
days = [date(year, month, day) for day in range(1, num_days+1)]
|
|
|
|
if obj == 'updates':
|
|
title = "Updates - " + str(month) + "/" + str(year)
|
|
js_labels = "'Up-to-date (%)', 'Need update (%)', 'Outdated (%)', 'Unknown (%)'"
|
|
title1 = "Updates status repartition (1 month)"
|
|
title2 = "Updates statistics repartition (1 month)"
|
|
fa_icon = "refresh"
|
|
elif obj == 'uptime':
|
|
title = "Uptime - " + str(month) + "/" + str(year)
|
|
js_labels = "'Rebooted recently (%)', 'Need a reboot (%)', 'Never rebooted (%)', 'Unknown (%)'"
|
|
title1 = "Uptime status repartition (1 month)"
|
|
title2 = "Uptime statistics repartition (1 month)"
|
|
fa_icon = "refresh"
|
|
elif obj == 'os':
|
|
title = "OS status - " + str(month) + "/" + str(year)
|
|
js_labels = "'Maintained (%)', 'End of support soon (%)', 'Out of support (%)', 'Unknown (%)'"
|
|
title1 = "OS status repartition (1 month)"
|
|
title2 = ""
|
|
fa_icon = "refresh"
|
|
else:
|
|
return HttpResponseNotFound('Page not found')
|
|
|
|
# to process mean
|
|
pc_ok_list = []
|
|
|
|
# legends
|
|
if obj == 'updates':
|
|
legend1 = ["#updates < 20", "20 < #updates < 100", "100 < #updates", "unknown number of updates"]
|
|
legend2 = ["mean #updates", "median #updates"]
|
|
elif obj == 'uptime':
|
|
legend1 = ["uptime < 100 days", "100 < uptime < 365 days", "365 days < uptime", "unknown uptime"]
|
|
legend2 = ["mean uptime", "median uptime"]
|
|
elif obj == 'os':
|
|
legend1 = ["OS maintained for more than 1 year", "OS unmaintained in less than 1 year", "OS unmaintained", "unknown OS"]
|
|
legend2 = []
|
|
|
|
# generate data (javascript) for each day
|
|
some_results = False
|
|
for day in days:
|
|
status = ServerStatus.objects.filter(date=day)
|
|
if status.count() != 0:
|
|
some_results = True
|
|
nb_tot = status.count() + 1
|
|
if obj == 'updates':
|
|
# pourcentage
|
|
pc_ok = len([s for s in status if s.updates_status()==1]) * 100 / nb_tot
|
|
pc_warn = len([s for s in status if s.updates_status()==2]) * 100 / nb_tot
|
|
pc_crit = len([s for s in status if s.updates_status()==3]) * 100 / nb_tot
|
|
pc_unk = len([s for s in status if s.updates_status()==0]) * 100 / nb_tot
|
|
# mean / median
|
|
updates_list = [upd for upd in status.values_list('updates', flat=True) if upd]
|
|
mean_val = mean(updates_list or [0])
|
|
median_val = median(updates_list or [0])
|
|
# to process mean
|
|
pc_ok_list = pc_ok_list + [pc_ok]
|
|
elif obj == 'uptime':
|
|
# pourcentage
|
|
pc_ok = len([s for s in status if s.uptime_status()==1]) * 100 / nb_tot
|
|
pc_warn = len([s for s in status if s.uptime_status()==2]) * 100 / nb_tot
|
|
pc_crit = len([s for s in status if s.uptime_status()==3]) * 100 / nb_tot
|
|
pc_unk = len([s for s in status if s.uptime_status()==0]) * 100 / nb_tot
|
|
# mean / median
|
|
uptime_list = [upd for upd in status.values_list('uptime', flat=True) if upd]
|
|
mean_val = mean(uptime_list or [0])
|
|
median_val = median(uptime_list or [0])
|
|
elif obj == 'os':
|
|
# pourcentage
|
|
pc_ok = len([s for s in status if s.os_status()==1]) * 100 / nb_tot
|
|
pc_warn = len([s for s in status if s.os_status()==2]) * 100 / nb_tot
|
|
pc_crit = len([s for s in status if s.os_status()==3]) * 100 / nb_tot
|
|
pc_unk = len([s for s in status if s.os_status()==0]) * 100 / nb_tot
|
|
|
|
pc_js_data = pc_js_data + """
|
|
{
|
|
period: '%s',
|
|
ok: %.2f,
|
|
warn: %.2f,
|
|
crit: %.2f,
|
|
unknown: %.2f
|
|
},""" % (day, pc_ok, pc_warn, pc_crit, pc_unk)
|
|
|
|
if obj == 'os':
|
|
mean_js_data = False
|
|
else:
|
|
mean_js_data = mean_js_data + """
|
|
{
|
|
period: '%s',
|
|
mean: %d,
|
|
median: %d
|
|
},""" % (day, mean_val, median_val)
|
|
|
|
mean_pc_ok = None
|
|
if obj == 'updates':
|
|
# pc_ok_list may be empty
|
|
if len(pc_ok_list) > 0:
|
|
mean_pc_ok = mean(pc_ok_list)
|
|
else:
|
|
mean_pc_ok = 0
|
|
|
|
return render(request, 'history.html',
|
|
{
|
|
'some_results': some_results,
|
|
'pc_js_data': pc_js_data,
|
|
'mean_js_data': mean_js_data,
|
|
'title': title,
|
|
'title1': title1,
|
|
'title2': title2,
|
|
'legend1': legend1,
|
|
'legend2': legend2,
|
|
'js_labels': js_labels,
|
|
'fa_icon': 'refresh',
|
|
'obj': obj,
|
|
'year': year,
|
|
'month': "%02d" % month,
|
|
'months': ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'],
|
|
'years': [datetime.now().year-1, datetime.now().year],
|
|
'mean_pc_ok': mean_pc_ok,
|
|
})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## PURGE ALL
|
|
## Purge everything in database. Dangerous!
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def purge_all(request):
|
|
Server.objects.all().delete()
|
|
ServerStatus.objects.all().delete()
|
|
Os.objects.all().delete()
|
|
PackageStatus.objects.all().delete()
|
|
Group.objects.all().delete()
|
|
|
|
content = "<div class='alert alert-danger' role='alert'>Everything has been purged.</div>"
|
|
|
|
return render(request, 'generic.html', {content: content})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## PURGE
|
|
## Purge statuses for one specific date
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def purge_statuses_by_date(request, year, month, day):
|
|
status = ServerStatus.objects.filter(date=date(int(year), int(month), int(day)))
|
|
if status:
|
|
content = "<div class='alert alert-success' role='alert'>Status for %s-%s-%s have been removed.</div>" % (year, month, day)
|
|
status.delete()
|
|
else:
|
|
content = "<div class='alert alert-danger' role='alert'>No server status for this date: %s-%s-%s.</div>" % (year, month, day)
|
|
|
|
return render(request, 'generic.html', { 'content': content, })
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## PURGE PACKAGES
|
|
## Purge all packages
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def purge_packages(request):
|
|
Server.objects.all().delete()
|
|
PackageStatus.objects.all().delete()
|
|
Os.objects.all().delete()
|
|
|
|
content = "<div class='alert alert-danger' role='alert'>Packages have been purged.</div>"
|
|
|
|
return render(request, 'generic.html', {content: content})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## IMPORT CSV
|
|
## Import statuses list from CSV
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def import_csv(request, year, month, day):
|
|
# Reminder: the imported file must be in this format:
|
|
# myserver.fr;Ubuntu;16;28;12
|
|
# | | | | |----- 4: uptime in days
|
|
# | | | |-------- 3: number of available updates
|
|
# | | |----------- 2: OS major version (string). Must match with OS model data
|
|
# | |---------------- 1: OS distribution. Must match with OS model data
|
|
# |-------------------------- 0: hostname (non-null required)
|
|
|
|
# TODO: import multiple
|
|
|
|
# TODO: cache hosts?
|
|
|
|
os_unknown, _ = Os.objects.get_or_create(distribution='Unknown', version='1')
|
|
# and retrieve all servers
|
|
servers_all = [[s.hostname, s] for s in Server.objects.select_related('os').all()]
|
|
|
|
# month and day on two digits
|
|
month = "%02d" % int(month)
|
|
day = "%02d" % int(day)
|
|
|
|
file_name = "%s-%s-%s.csv" % (year, month, day)
|
|
file_path = path.join(settings.RESULT_DIR, file_name)
|
|
if path.isfile(file_path):
|
|
content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
|
|
f_o = open(file_path, "rt")
|
|
f_r = reader(f_o, delimiter=';')
|
|
|
|
# first, delete old status
|
|
ServerStatus.objects.filter(date=date(int(year), int(month), int(day))).delete()
|
|
|
|
# loop on server list
|
|
for row in f_r:
|
|
if len(row) > 2:
|
|
row_hostname = row[0]
|
|
row_os_distribution = row[1]
|
|
row_os_version = row[2]
|
|
row_updates = row[3] if row[3] else None
|
|
row_uptime = row[4] if row[4] else None
|
|
|
|
# get OS
|
|
cache_key_os = "os_%s_%s" % (row_os_distribution, row_os_version)
|
|
cache_ttl = 1800
|
|
current_os = cache.get(cache_key_os)
|
|
if not current_os:
|
|
try:
|
|
current_os = Os.objects.get(distribution=row_os_distribution, version=row_os_version)
|
|
except Os.DoesNotExist:
|
|
current_os = os_unknown
|
|
cache.set(cache_key_os, current_os, cache_ttl)
|
|
|
|
|
|
# get server if exists
|
|
found_server = False
|
|
for s in servers_all:
|
|
if s[0] == row_hostname:
|
|
found_server = True
|
|
current_s = s[1]
|
|
pass
|
|
# if not, create it
|
|
if not found_server:
|
|
current_s = Server(hostname=row_hostname)
|
|
|
|
# set or update the server OS if needed
|
|
if current_s.os != current_os:
|
|
current_s.os = current_os
|
|
current_s.save()
|
|
|
|
current_st = ServerStatus(
|
|
date = date(int(year), int(month), int(day)),
|
|
server = current_s
|
|
)
|
|
current_st.updates = row_updates
|
|
current_st.uptime = row_uptime
|
|
try:
|
|
current_st.save()
|
|
except IntegrityError:
|
|
content = content + "<br><div class='alert alert-warning' role='alert'><b>Warning:</b> Multiple status for %s</div>" % row_hostname
|
|
|
|
else:
|
|
content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path
|
|
|
|
return render(request, 'generic.html', {
|
|
'content': content,
|
|
})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## IMPORT CSV PACKAGES
|
|
## Import packages list from CSV
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def import_csv_packages(request, year, month, day):
|
|
# Reminder: the imported file must be in this format:
|
|
# myserver.fr;nompackage;version
|
|
# | | |----- 2: version
|
|
# | |---------------- 1: package name
|
|
# |-------------------------- 0: hostname (non-null required)
|
|
|
|
# TODO: improve import: don't look for server for each line, but loop csv
|
|
# before to group by host
|
|
|
|
# TODO: cache host?
|
|
|
|
# retrieve all servers
|
|
servers_all = [[s.hostname, s] for s in Server.objects.select_related('os').all()]
|
|
|
|
# month and day on two digits
|
|
month = "%02d" % int(month)
|
|
day = "%02d" % int(day)
|
|
|
|
file_name = "%s-%s-%s.csv" % (year, month, day)
|
|
file_path = path.join(settings.RESULT_PACKAGES_DIR, file_name)
|
|
if path.isfile(file_path):
|
|
content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
|
|
f_o = open(file_path, "rt")
|
|
f_r = reader(f_o, delimiter=';')
|
|
|
|
# first, delete old packages
|
|
PackageStatus.objects.all().delete()
|
|
|
|
# loop on server list
|
|
for row in f_r:
|
|
|
|
if len(row) == 3:
|
|
row_hostname = row[0]
|
|
row_packages_name = row[1]
|
|
row_packages_version = row[2]
|
|
|
|
# get server if exists
|
|
found_server = False
|
|
for s in servers_all:
|
|
if s[0] == row_hostname:
|
|
found_server = True
|
|
current_s = s[1]
|
|
pass
|
|
|
|
if found_server:
|
|
package_status = PackageStatus(
|
|
server = current_s,
|
|
package_name = row_packages_name,
|
|
package_version = row_packages_version
|
|
|
|
)
|
|
|
|
package_status.save()
|
|
|
|
else:
|
|
content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path
|
|
|
|
return render(request, 'generic.html', {
|
|
'content': content,
|
|
})
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## MANAGE
|
|
## Manage statuses and files
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def manage(request):
|
|
# existing dates?
|
|
stat_dates = sorted(list(set([s[0] for s in ServerStatus.objects.values_list('date')])))
|
|
|
|
# available files?
|
|
files_dates = []
|
|
for file_name in sorted(listdir(settings.RESULT_DIR)):
|
|
# is it xxxx-xx-xx.csv?
|
|
d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
|
|
# group(1) = 2019, group(2) = 09, group(3) = 20
|
|
if d and stat(path.join(settings.RESULT_DIR, file_name)).st_size > 0:
|
|
files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))
|
|
|
|
all_dates = sorted(list(set(stat_dates + files_dates)))
|
|
|
|
all_dates_status = []
|
|
for d in all_dates:
|
|
if d in stat_dates:
|
|
all_dates_status.append([d, True])
|
|
else:
|
|
all_dates_status.append([d, False])
|
|
|
|
return render(request, 'manage.html', {
|
|
'all_dates_status': reversed(all_dates_status),
|
|
})
|
|
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## MANAGE PACKAGES
|
|
## Manage packages and files
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def manage_packages(request):
|
|
# available files?
|
|
files_dates = []
|
|
for file_name in sorted(listdir(settings.RESULT_PACKAGES_DIR)):
|
|
# is it xxxx-xx-xx.csv?
|
|
d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
|
|
# group(1) = 2019, group(2) = 09, group(3) = 20
|
|
if d and stat(path.join(settings.RESULT_PACKAGES_DIR, file_name)).st_size > 0:
|
|
files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))
|
|
|
|
all_dates = sorted(files_dates)
|
|
|
|
return render(request, 'manage-packages.html', {
|
|
'all_dates': reversed(all_dates),
|
|
})
|
|
|
|
## -----------------------------------------------------------------------------
|
|
## UPLOAD FILE
|
|
## Upload csv file
|
|
## -----------------------------------------------------------------------------
|
|
|
|
@login_required
|
|
def upload_csv_results(request):
|
|
message = 'File must be name YYYY-MM-DD.csv'
|
|
# Handle file upload
|
|
if request.method == 'POST':
|
|
form = DocumentForm(request.POST, request.FILES)
|
|
if form.is_valid():
|
|
newdoc = Document_Servers(docfile=request.FILES['docfile'])
|
|
newdoc.save()
|
|
|
|
# Redirect to the document list after POST
|
|
return redirect('manage')
|
|
else:
|
|
message = 'The form is not valid. Fix the following error:'
|
|
else:
|
|
form = DocumentForm() # An empty, unbound form
|
|
|
|
# Load documents for the list page
|
|
documents = Document_Servers.objects.all()
|
|
|
|
# Render list page with the documents and the form
|
|
context = {'documents': documents, 'form': form, 'message': message}
|
|
return render(request, 'manage.html', context)
|