2022-08-19 10:37:38 +02:00

648 lines
25 KiB
Python

# python
from os import listdir, path, stat
from re import match
from datetime import date, datetime, timedelta
from calendar import monthrange
from csv import reader
from statistics import mean, median
from yaml import load as yaml_load
from requests import get as requests_get
# django
from django.shortcuts import render, get_object_or_404, redirect
from django.http import HttpResponse, HttpResponseNotFound
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.core.cache import cache
from django.db import IntegrityError
from django.contrib.auth.decorators import login_required
# project
from .models import Os, Group, Server, ServerStatus, PackageStatus, Team, Document_Servers, Document_Packages
from .forms import DocumentForm
## -----------------------------------------------------------------------------
## INDEX
## Home page
## -----------------------------------------------------------------------------
@login_required
def index(request):
return render(request, 'index.html', {})
## -----------------------------------------------------------------------------
## SERVER LIST
## Show servers, their status, number of updates...
## -----------------------------------------------------------------------------
@login_required
def server_list(request, year=None, month=None, day=None, group=None, team=None):
# TODO: use date.today?
now = datetime.now()
if not year or not month or not day:
year = now.year
month = now.month
day = now.day
year = int(year)
month = int(month)
day = int(day)
# check if this is a day
try:
current_date = date(year, month, day)
except ValueError:
return HttpResponseNotFound('Page not found')
# retrieve the date of the fist results before and after current date
previous_result_date = None
previous_result = ServerStatus.objects.filter(date__lt=current_date).order_by('date').last()
if previous_result is not None:
previous_result_date = previous_result.date
next_result_date = None
next_result = ServerStatus.objects.filter(date__gt=current_date).order_by('date').first()
if next_result is not None:
next_result_date = next_result.date
results_date = current_date
if group:
group = get_object_or_404(Group, name=group)
status_list = ServerStatus.objects.filter(date=current_date, server__group=group).order_by('server__hostname')
if previous_result and not status_list:
status_list = ServerStatus.objects.filter(date=previous_result_date, server__group=group).order_by('server__hostname')
results_date = previous_result_date
elif team:
team = get_object_or_404(Team, color=team)
status_list = ServerStatus.objects.filter(date=current_date, server__team=team).order_by('server__hostname')
if previous_result and not status_list:
status_list = ServerStatus.objects.filter(date=previous_result_date, server__team=team).order_by('server__hostname')
results_date = previous_result_date
else:
# status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname')
status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname').select_related('server', 'server__group', 'server__os')
if previous_result and not status_list:
status_list = ServerStatus.objects.filter(date=previous_result_date).order_by('server__hostname').select_related('server', 'server__group', 'server__os')
results_date = previous_result_date
if not status_list:
return render(request, 'generic.html', {
'content': '<p>No results found.</p>',
})
return render(request, 'server-list.html',
{
'group': group,
'status_list': status_list,
'results_date': results_date,
'previous_result_date': previous_result_date,
'next_result_date': next_result_date,
})
## -----------------------------------------------------------------------------
## PACKAGES LIST
## Show package list
## -----------------------------------------------------------------------------
@login_required
def packages_list(request):
packages_list_all = [p.package_name for p in PackageStatus.objects.all()]
packages_list = []
for p in set(packages_list_all):
packages_list.append([p, packages_list_all.count(p)])
return render(request, 'packages-list.html', { 'packages_list': packages_list, })
## -----------------------------------------------------------------------------
## SHOW PACKAGE
## Show servers and versions of a package
## -----------------------------------------------------------------------------
@login_required
def packages(request, package=None , hostname=None):
# what do we want?
if package is not None:
# show all servers that have this package
packages = PackageStatus.objects.filter(package_name=package)
else:
if hostname is not None:
# show package for one host
packages = PackageStatus.objects.filter(server__hostname=hostname)
else:
# show all packages for all hosts
packages = PackageStatus.objects.all()
return render(request, 'packages.html', { 'packages': packages, })
## -----------------------------------------------------------------------------
## OS STATISTICS
## Show percentage for each distrib / version, and a nice chart
## -----------------------------------------------------------------------------
@login_required
def os_statistics(request):
# get last date for which we have stats
last_date = ServerStatus.objects.all().order_by('date').last().date
current_st_list = ServerStatus.objects.filter(date=last_date)
nb_servers = len(current_st_list)
os_list = Os.objects.all()
os_stat = []
js_data = ""
if last_date is not None:
# count how many server we have for each distribution
for os in os_list:
nb = len(current_st_list.filter(server__os=os))
if nb > 0:
os_stat.append([str(os), nb, nb * 100 / nb_servers])
# for the chart
js_data = js_data + """
{
label: "%s",
value: %d
},""" % (str(os), nb)
return render(request, 'os-statistics.html', {
'js_data': js_data,
'os_stat': os_stat,
'last_date': last_date,
})
## -----------------------------------------------------------------------------
## HISTORY
## Show graphs for a month: updates or uptime or os-eol
## -----------------------------------------------------------------------------
@login_required
def history(request, obj, year=False, month=False):
if not year:
year = datetime.now().year
if not month:
month = datetime.now().month
if int(month) == int(datetime.now().month):
current_month = True
else:
current_month = False
year = int(year)
month = int(month)
pc_js_data = ""
mean_js_data = ""
if current_month:
num_days = 30
days = [datetime.now() - timedelta(days=n) for n in range(num_days, -1, -1)]
else:
num_days = monthrange(year,month)[1]
days = [date(year, month, day) for day in range(1, num_days+1)]
if obj == 'updates':
title = "Updates - " + str(month) + "/" + str(year)
js_labels = "'Up-to-date (%)', 'Need update (%)', 'Outdated (%)', 'Unknown (%)'"
title1 = "Updates status repartition (1 month)"
title2 = "Updates statistics repartition (1 month)"
fa_icon = "refresh"
elif obj == 'uptime':
title = "Uptime - " + str(month) + "/" + str(year)
js_labels = "'Rebooted recently (%)', 'Need a reboot (%)', 'Never rebooted (%)', 'Unknown (%)'"
title1 = "Uptime status repartition (1 month)"
title2 = "Uptime statistics repartition (1 month)"
fa_icon = "refresh"
elif obj == 'os':
title = "OS status - " + str(month) + "/" + str(year)
js_labels = "'Maintained (%)', 'End of support soon (%)', 'Out of support (%)', 'Unknown (%)'"
title1 = "OS status repartition (1 month)"
title2 = ""
fa_icon = "refresh"
else:
return HttpResponseNotFound('Page not found')
# to process mean
pc_ok_list = []
# legends
if obj == 'updates':
legend1 = ["#updates < 20", "20 < #updates < 100", "100 < #updates", "unknown number of updates"]
legend2 = ["mean #updates", "median #updates"]
elif obj == 'uptime':
legend1 = ["uptime < 100 days", "100 < uptime < 365 days", "365 days < uptime", "unknown uptime"]
legend2 = ["mean uptime", "median uptime"]
elif obj == 'os':
legend1 = ["OS maintained for more than 1 year", "OS unmaintained in less than 1 year", "OS unmaintained", "unknown OS"]
legend2 = []
# generate data (javascript) for each day
some_results = False
for day in days:
status = ServerStatus.objects.filter(date=day)
if status.count() != 0:
some_results = True
nb_tot = status.count() + 1
if obj == 'updates':
# pourcentage
pc_ok = len([s for s in status if s.updates_status()==1]) * 100 / nb_tot
pc_warn = len([s for s in status if s.updates_status()==2]) * 100 / nb_tot
pc_crit = len([s for s in status if s.updates_status()==3]) * 100 / nb_tot
pc_unk = len([s for s in status if s.updates_status()==0]) * 100 / nb_tot
# mean / median
updates_list = [upd for upd in status.values_list('updates', flat=True) if upd]
mean_val = mean(updates_list or [0])
median_val = median(updates_list or [0])
# to process mean
pc_ok_list = pc_ok_list + [pc_ok]
elif obj == 'uptime':
# pourcentage
pc_ok = len([s for s in status if s.uptime_status()==1]) * 100 / nb_tot
pc_warn = len([s for s in status if s.uptime_status()==2]) * 100 / nb_tot
pc_crit = len([s for s in status if s.uptime_status()==3]) * 100 / nb_tot
pc_unk = len([s for s in status if s.uptime_status()==0]) * 100 / nb_tot
# mean / median
uptime_list = [upd for upd in status.values_list('uptime', flat=True) if upd]
mean_val = mean(uptime_list or [0])
median_val = median(uptime_list or [0])
elif obj == 'os':
# pourcentage
pc_ok = len([s for s in status if s.os_status()==1]) * 100 / nb_tot
pc_warn = len([s for s in status if s.os_status()==2]) * 100 / nb_tot
pc_crit = len([s for s in status if s.os_status()==3]) * 100 / nb_tot
pc_unk = len([s for s in status if s.os_status()==0]) * 100 / nb_tot
pc_js_data = pc_js_data + """
{
period: '%s',
ok: %.2f,
warn: %.2f,
crit: %.2f,
unknown: %.2f
},""" % (day, pc_ok, pc_warn, pc_crit, pc_unk)
if obj == 'os':
mean_js_data = False
else:
mean_js_data = mean_js_data + """
{
period: '%s',
mean: %d,
median: %d
},""" % (day, mean_val, median_val)
mean_pc_ok = None
if obj == 'updates':
# pc_ok_list may be empty
if len(pc_ok_list) > 0:
mean_pc_ok = mean(pc_ok_list)
else:
mean_pc_ok = 0
return render(request, 'history.html',
{
'some_results': some_results,
'pc_js_data': pc_js_data,
'mean_js_data': mean_js_data,
'title': title,
'title1': title1,
'title2': title2,
'legend1': legend1,
'legend2': legend2,
'js_labels': js_labels,
'fa_icon': 'refresh',
'obj': obj,
'year': year,
'month': "%02d" % month,
'months': ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'],
'years': [datetime.now().year-1, datetime.now().year],
'mean_pc_ok': mean_pc_ok,
})
## -----------------------------------------------------------------------------
## PURGE ALL
## Purge everything in database. Dangerous!
## -----------------------------------------------------------------------------
@login_required
def purge_all(request):
Server.objects.all().delete()
ServerStatus.objects.all().delete()
Os.objects.all().delete()
PackageStatus.objects.all().delete()
Group.objects.all().delete()
content = "<div class='alert alert-danger' role='alert'>Everything has been purged.</div>"
return render(request, 'generic.html', {content: content})
## -----------------------------------------------------------------------------
## PURGE
## Purge statuses for one specific date
## -----------------------------------------------------------------------------
@login_required
def purge_statuses_by_date(request, year, month, day):
status = ServerStatus.objects.filter(date=date(int(year), int(month), int(day)))
if status:
content = "<div class='alert alert-success' role='alert'>Status for %s-%s-%s have been removed.</div>" % (year, month, day)
status.delete()
else:
content = "<div class='alert alert-danger' role='alert'>No server status for this date: %s-%s-%s.</div>" % (year, month, day)
return render(request, 'generic.html', { 'content': content, })
## -----------------------------------------------------------------------------
## PURGE PACKAGES
## Purge all packages
## -----------------------------------------------------------------------------
@login_required
def purge_packages(request):
Server.objects.all().delete()
PackageStatus.objects.all().delete()
Os.objects.all().delete()
content = "<div class='alert alert-danger' role='alert'>Packages have been purged.</div>"
return render(request, 'generic.html', {content: content})
## -----------------------------------------------------------------------------
## IMPORT CSV
## Import statuses list from CSV
## -----------------------------------------------------------------------------
@login_required
def import_csv(request, year, month, day):
# Reminder: the imported file must be in this format:
# myserver.fr;Ubuntu;16;28;12
# | | | | |----- 4: uptime in days
# | | | |-------- 3: number of available updates
# | | |----------- 2: OS major version (string). Must match with OS model data
# | |---------------- 1: OS distribution. Must match with OS model data
# |-------------------------- 0: hostname (non-null required)
# TODO: import multiple
# TODO: cache hosts?
os_unknown, _ = Os.objects.get_or_create(distribution='Unknown', version='1')
# and retrieve all servers
servers_all = [[s.hostname, s] for s in Server.objects.select_related('os').all()]
# month and day on two digits
month = "%02d" % int(month)
day = "%02d" % int(day)
file_name = "%s-%s-%s.csv" % (year, month, day)
file_path = path.join(settings.RESULT_DIR, file_name)
if path.isfile(file_path):
content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
f_o = open(file_path, "rt")
f_r = reader(f_o, delimiter=';')
# first, delete old status
ServerStatus.objects.filter(date=date(int(year), int(month), int(day))).delete()
# loop on server list
for row in f_r:
if len(row) > 2:
row_hostname = row[0]
row_os_distribution = row[1]
row_os_version = row[2]
row_updates = row[3] if row[3] else None
row_uptime = row[4] if row[4] else None
# get OS
cache_key_os = "os_%s_%s" % (row_os_distribution, row_os_version)
cache_ttl = 1800
current_os = cache.get(cache_key_os)
if not current_os:
try:
current_os = Os.objects.get(distribution=row_os_distribution, version=row_os_version)
except Os.DoesNotExist:
current_os = os_unknown
cache.set(cache_key_os, current_os, cache_ttl)
# get server if exists
found_server = False
for s in servers_all:
if s[0] == row_hostname:
found_server = True
current_s = s[1]
pass
# if not, create it
if not found_server:
current_s = Server(hostname=row_hostname)
# set or update the server OS if needed
if current_s.os != current_os:
current_s.os = current_os
current_s.save()
current_st = ServerStatus(
date = date(int(year), int(month), int(day)),
server = current_s
)
current_st.updates = row_updates
current_st.uptime = row_uptime
try:
current_st.save()
except IntegrityError:
content = content + "<br><div class='alert alert-warning' role='alert'><b>Warning:</b> Multiple status for %s</div>" % row_hostname
else:
content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path
return render(request, 'generic.html', {
'content': content,
})
## -----------------------------------------------------------------------------
## IMPORT CSV PACKAGES
## Import packages list from CSV
## -----------------------------------------------------------------------------
@login_required
def import_csv_packages(request, year, month, day):
# Reminder: the imported file must be in this format:
# myserver.fr;nompackage;version
# | | |----- 2: version
# | |---------------- 1: package name
# |-------------------------- 0: hostname (non-null required)
# TODO: improve import: don't look for server for each line, but loop csv
# before to group by host
# TODO: cache host?
# retrieve all servers
servers_all = [[s.hostname, s] for s in Server.objects.select_related('os').all()]
# month and day on two digits
month = "%02d" % int(month)
day = "%02d" % int(day)
file_name = "%s-%s-%s.csv" % (year, month, day)
file_path = path.join(settings.RESULT_PACKAGES_DIR, file_name)
if path.isfile(file_path):
content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
f_o = open(file_path, "rt")
f_r = reader(f_o, delimiter=';')
# first, delete old packages
PackageStatus.objects.all().delete()
# loop on server list
for row in f_r:
if len(row) == 3:
row_hostname = row[0]
row_packages_name = row[1]
row_packages_version = row[2]
# get server if exists
found_server = False
for s in servers_all:
if s[0] == row_hostname:
found_server = True
current_s = s[1]
pass
if found_server:
package_status = PackageStatus(
server = current_s,
package_name = row_packages_name,
package_version = row_packages_version
)
package_status.save()
else:
content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path
return render(request, 'generic.html', {
'content': content,
})
## -----------------------------------------------------------------------------
## MANAGE
## Manage statuses and files
## -----------------------------------------------------------------------------
@login_required
def manage(request):
# existing dates?
stat_dates = sorted(list(set([s[0] for s in ServerStatus.objects.values_list('date')])))
# available files?
files_dates = []
for file_name in sorted(listdir(settings.RESULT_DIR)):
# is it xxxx-xx-xx.csv?
d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
# group(1) = 2019, group(2) = 09, group(3) = 20
if d and stat(path.join(settings.RESULT_DIR, file_name)).st_size > 0:
files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))
all_dates = sorted(list(set(stat_dates + files_dates)))
all_dates_status = []
for d in all_dates:
if d in stat_dates:
all_dates_status.append([d, True])
else:
all_dates_status.append([d, False])
return render(request, 'manage.html', {
'all_dates_status': reversed(all_dates_status),
})
## -----------------------------------------------------------------------------
## MANAGE PACKAGES
## Manage packages and files
## -----------------------------------------------------------------------------
@login_required
def manage_packages(request):
# available files?
files_dates = []
for file_name in sorted(listdir(settings.RESULT_PACKAGES_DIR)):
# is it xxxx-xx-xx.csv?
d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
# group(1) = 2019, group(2) = 09, group(3) = 20
if d and stat(path.join(settings.RESULT_PACKAGES_DIR, file_name)).st_size > 0:
files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))
all_dates = sorted(files_dates)
return render(request, 'manage-packages.html', {
'all_dates': reversed(all_dates),
})
## -----------------------------------------------------------------------------
## UPLOAD FILE
## Upload csv file for servers informations
## -----------------------------------------------------------------------------
@login_required
def upload_csv_results(request):
message = 'File must be name YYYY-MM-DD.csv'
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document_Servers(docfile=request.FILES['docfile'])
newdoc.save()
# Redirect to the document list after POST
return redirect('manage')
else:
message = 'The form is not valid. Fix the following error:'
else:
form = DocumentForm() # An empty, unbound form
# Load documents for the list page
documents = Document_Servers.objects.all()
# Render list page with the documents and the form
context = {'documents': documents, 'form': form, 'message': message}
return render(request, 'manage.html', context)
## -----------------------------------------------------------------------------
## UPLOAD FILE
## Upload csv file for packages informations
## -----------------------------------------------------------------------------
@login_required
def upload_csv_results_packages(request):
message = 'File must be name YYYY-MM-DD.csv'
# Handle file upload
if request.method == 'POST':
form = DocumentForm(request.POST, request.FILES)
if form.is_valid():
newdoc = Document_Packages(docfile=request.FILES['docfile'])
newdoc.save()
# Redirect to the document list after POST
return redirect('manage-packages')
else:
message = 'The form is not valid. Fix the following error:'
else:
form = DocumentForm() # An empty, unbound form
# Load documents for the list page
documents = Document_Packages.objects.all()
# Render list page with the documents and the form
context = {'documents': documents, 'form': form, 'message': message}
return render(request, 'manage-packages.html', context)