# python
from os                         import listdir, path, stat
from re                         import match
from datetime                   import date, datetime, timedelta
from calendar                   import monthrange
from csv                        import reader
from statistics                 import mean, median
from yaml                       import load as yaml_load
from requests                   import get as requests_get
# django
from django.shortcuts               import render, get_object_or_404
from django.http                    import HttpResponse, HttpResponseNotFound
from django.conf                    import settings
from django.core.exceptions         import ObjectDoesNotExist
from django.core.cache              import cache
from django.db                      import IntegrityError
from django.contrib.auth.decorators import login_required
# project
from .models                    import Os, Group, Server, ServerStatus, PackageStatus, Team


## -----------------------------------------------------------------------------
## INDEX
## Home page
## -----------------------------------------------------------------------------

@login_required
def index(request):
    return render(request, 'index.html', {})


## -----------------------------------------------------------------------------
## SERVER LIST
## Show servers, their status, number of updates...
## -----------------------------------------------------------------------------

@login_required
def server_list(request, year=None, month=None, day=None, group=None, team=None):
    # TODO: use date.today?
    now = datetime.now()
    if not year or not month or not day:
        year  = now.year
        month = now.month
        day   = now.day
    year  = int(year)
    month = int(month)
    day   = int(day)

    # check if this is a day
    try:
        current_date = date(year, month, day)
    except ValueError:
        return HttpResponseNotFound('Page not found')

    # retrieve the date of the fist results before and after current date
    previous_result_date = None
    previous_result = ServerStatus.objects.filter(date__lt=current_date).order_by('date').last()
    if previous_result is not None:
        previous_result_date = previous_result.date
    next_result_date = None
    next_result = ServerStatus.objects.filter(date__gt=current_date).order_by('date').first()
    if next_result is not None:
        next_result_date = next_result.date

    results_date = current_date
    if group:
        group = get_object_or_404(Group, name=group)
        status_list = ServerStatus.objects.filter(date=current_date, server__group=group).order_by('server__hostname')
        if previous_result and not status_list:
            status_list = ServerStatus.objects.filter(date=previous_result_date, server__group=group).order_by('server__hostname')
            results_date = previous_result_date
    elif team:
        team = get_object_or_404(Team, color=team)
        status_list = ServerStatus.objects.filter(date=current_date, server__team=team).order_by('server__hostname')
        if previous_result and not status_list:
            status_list = ServerStatus.objects.filter(date=previous_result_date, server__team=team).order_by('server__hostname')
            results_date = previous_result_date
    else:
        # status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname')
        status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname').select_related('server', 'server__group', 'server__os')
        if previous_result and not status_list:
            status_list = ServerStatus.objects.filter(date=previous_result_date).order_by('server__hostname').select_related('server', 'server__group', 'server__os')
            results_date = previous_result_date

    if not status_list:
        return render(request, 'generic.html', {
                'content': '<p>No results found.</p>',
            })

    return render(request, 'server-list.html',
                  {
                      'group':                group,
                      'status_list':          status_list,
                      'results_date':         results_date,
                      'previous_result_date': previous_result_date,
                      'next_result_date':     next_result_date,
                  })


## -----------------------------------------------------------------------------
## PACKAGES LIST
## Show package list
## -----------------------------------------------------------------------------

@login_required
def packages_list(request):
    packages_list_all = [p.package_name for p in PackageStatus.objects.all()]
    packages_list     = []

    for p in set(packages_list_all):
        packages_list.append([p, packages_list_all.count(p)])

    return render(request, 'packages-list.html', { 'packages_list': packages_list, })


## -----------------------------------------------------------------------------
## SHOW PACKAGE
## Show servers and versions of a package
## -----------------------------------------------------------------------------

@login_required
def packages(request, package=None , hostname=None):
    # what do we want?
    if package is not None:
        # show all servers that have this package
        packages = PackageStatus.objects.filter(package_name=package)
    else:
        if hostname is not None:
            # show package for one host
            packages = PackageStatus.objects.filter(server__hostname=hostname)
        else:
            # show all packages for all hosts
            packages = PackageStatus.objects.all()

    return render(request, 'packages.html', { 'packages': packages, })


## -----------------------------------------------------------------------------
## OS STATISTICS
## Show percentage for each distrib / version, and a nice chart
## -----------------------------------------------------------------------------

@login_required
def os_statistics(request):
    # get last date for which we have stats
    last_date       = ServerStatus.objects.all().order_by('date').last().date
    current_st_list = ServerStatus.objects.filter(date=last_date)
    nb_servers      = len(current_st_list)
    os_list         = Os.objects.all()
    os_stat         = []
    js_data         = ""

    if last_date is not None:
        # count how many server we have for each distribution
        for os in os_list:
            nb = len(current_st_list.filter(server__os=os))

            if nb > 0:
                os_stat.append([str(os), nb, nb * 100 / nb_servers])
                # for the chart
                js_data = js_data + """
                    {
                        label: "%s",
                        value:  %d
                    },""" % (str(os), nb)

    return render(request, 'os-statistics.html', {
            'js_data':   js_data,
            'os_stat':   os_stat,
            'last_date': last_date,
        })


## -----------------------------------------------------------------------------
## HISTORY
## Show graphs for a month: updates or uptime or os-eol
## -----------------------------------------------------------------------------

@login_required
def history(request, obj, year=False, month=False):
    if not year:
        year = datetime.now().year
    if not month:
        month = datetime.now().month

    if int(month) == int(datetime.now().month):
        current_month = True
    else:
        current_month = False

    year  = int(year)
    month = int(month)

    pc_js_data   = ""
    mean_js_data = ""

    if current_month:
        num_days = 30
        days     = [datetime.now() - timedelta(days=n) for n in range(num_days, -1, -1)]
    else:
        num_days = monthrange(year,month)[1]
        days     = [date(year, month, day) for day in range(1, num_days+1)]

    if obj == 'updates':
        title     = "Updates - " + str(month) + "/" + str(year)
        js_labels = "'Up-to-date (%)', 'Need update (%)', 'Outdated (%)', 'Unknown (%)'"
        title1    = "Updates status repartition (1 month)"
        title2    = "Updates statistics repartition (1 month)"
        fa_icon   = "refresh"
    elif obj == 'uptime':
        title     = "Uptime - " + str(month) + "/" + str(year)
        js_labels = "'Rebooted recently (%)', 'Need a reboot (%)', 'Never rebooted (%)', 'Unknown (%)'"
        title1    = "Uptime status repartition (1 month)"
        title2    = "Uptime statistics repartition (1 month)"
        fa_icon   = "refresh"
    elif obj == 'os':
        title     = "OS status - " + str(month) + "/" + str(year)
        js_labels = "'Maintained (%)', 'End of support soon (%)', 'Out of support (%)', 'Unknown (%)'"
        title1    = "OS status repartition (1 month)"
        title2    = ""
        fa_icon   = "refresh"
    else:
        return HttpResponseNotFound('Page not found')

    # to process mean
    pc_ok_list = []

    # legends
    if obj == 'updates':
        legend1 = ["#updates < 20", "20 < #updates < 100", "100 < #updates", "unknown number of updates"]
        legend2 = ["mean #updates", "median #updates"]
    elif obj == 'uptime':
        legend1 = ["uptime < 100 days", "100 < uptime < 365 days", "365 days < uptime", "unknown uptime"]
        legend2 = ["mean uptime", "median uptime"]
    elif obj == 'os':
        legend1 = ["OS maintained for more than 1 year", "OS unmaintained in less than 1 year", "OS unmaintained", "unknown OS"]
        legend2 = []

    # generate data (javascript) for each day
    some_results = False
    for day in days:
        status = ServerStatus.objects.filter(date=day)
        if status.count() != 0:
            some_results = True
            nb_tot = status.count() + 1
            if obj == 'updates':
                # pourcentage
                pc_ok   = len([s for s in status if s.updates_status()==1]) * 100 / nb_tot
                pc_warn = len([s for s in status if s.updates_status()==2]) * 100 / nb_tot
                pc_crit = len([s for s in status if s.updates_status()==3]) * 100 / nb_tot
                pc_unk  = len([s for s in status if s.updates_status()==0]) * 100 / nb_tot
                # mean / median
                updates_list = [upd for upd in status.values_list('updates', flat=True) if upd]
                mean_val     = mean(updates_list or [0])
                median_val   = median(updates_list or [0])
                # to process mean
                pc_ok_list = pc_ok_list + [pc_ok]
            elif obj == 'uptime':
                # pourcentage
                pc_ok   = len([s for s in status if s.uptime_status()==1]) * 100 / nb_tot
                pc_warn = len([s for s in status if s.uptime_status()==2]) * 100 / nb_tot
                pc_crit = len([s for s in status if s.uptime_status()==3]) * 100 / nb_tot
                pc_unk  = len([s for s in status if s.uptime_status()==0]) * 100 / nb_tot
                # mean / median
                uptime_list = [upd for upd in status.values_list('uptime', flat=True) if upd]
                mean_val    = mean(uptime_list or [0])
                median_val  = median(uptime_list or [0])
            elif obj == 'os':
                # pourcentage
                pc_ok   = len([s for s in status if s.os_status()==1]) * 100 / nb_tot
                pc_warn = len([s for s in status if s.os_status()==2]) * 100 / nb_tot
                pc_crit = len([s for s in status if s.os_status()==3]) * 100 / nb_tot
                pc_unk  = len([s for s in status if s.os_status()==0]) * 100 / nb_tot

            pc_js_data = pc_js_data + """
            {
                period:  '%s',
                ok:      %.2f,
                warn:    %.2f,
                crit:    %.2f,
                unknown: %.2f
            },""" % (day, pc_ok, pc_warn, pc_crit, pc_unk)

            if obj == 'os':
                mean_js_data = False
            else:
                mean_js_data = mean_js_data + """
                {
                    period:  '%s',
                    mean:    %d,
                    median:  %d
                },""" % (day, mean_val, median_val)

    mean_pc_ok = None
    if obj == 'updates':
        # pc_ok_list may be empty
        if len(pc_ok_list) > 0:
            mean_pc_ok = mean(pc_ok_list)
        else:
            mean_pc_ok = 0

    return render(request, 'history.html',
                  {
                      'some_results': some_results,
                      'pc_js_data':   pc_js_data,
                      'mean_js_data': mean_js_data,
                      'title':        title,
                      'title1':       title1,
                      'title2':       title2,
                      'legend1':      legend1,
                      'legend2':      legend2,
                      'js_labels':    js_labels,
                      'fa_icon':      'refresh',
                      'obj':          obj,
                      'year':         year,
                      'month':        "%02d" % month,
                      'months':       ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'],
                      'years':        [datetime.now().year-1, datetime.now().year],
                      'mean_pc_ok':   mean_pc_ok,
                  })


## -----------------------------------------------------------------------------
## PURGE ALL
## Purge everything in database. Dangerous!
## -----------------------------------------------------------------------------

@login_required
def purge_all(request):
    Server.objects.all().delete()
    ServerStatus.objects.all().delete()
    Os.objects.all().delete()
    PackageStatus.objects.all().delete()
    Group.objects.all().delete()

    content = "<div class='alert alert-danger' role='alert'>Everything has been purged.</div>"

    return render(request, 'generic.html', {content: content})


## -----------------------------------------------------------------------------
## PURGE
## Purge statuses for one specific date
## -----------------------------------------------------------------------------

@login_required
def purge_statuses_by_date(request, year, month, day):
    status = ServerStatus.objects.filter(date=date(int(year), int(month), int(day)))
    if status:
        content = "<div class='alert alert-success' role='alert'>Status for %s-%s-%s have been removed.</div>" % (year, month, day)
        status.delete()
    else:
        content = "<div class='alert alert-danger' role='alert'>No server status for this date: %s-%s-%s.</div>" % (year, month, day)

    return render(request, 'generic.html', { 'content': content, })


## -----------------------------------------------------------------------------
## PURGE PACKAGES
## Purge all packages
## -----------------------------------------------------------------------------

@login_required
def purge_packages(request):
    Server.objects.all().delete()
    PackageStatus.objects.all().delete()
    Os.objects.all().delete()

    content = "<div class='alert alert-danger' role='alert'>Packages have been purged.</div>"

    return render(request, 'generic.html', {content: content})


## -----------------------------------------------------------------------------
## IMPORT CSV
## Import statuses list from CSV
## -----------------------------------------------------------------------------

@login_required
def import_csv(request, year, month, day):
    # Reminder: the imported file must be in this format:
    # myserver.fr;Ubuntu;16;28;12;UNK
    #     |         |    |  |  |   |- 5: auto updates status
    #     |         |    |  |  |----- 4: uptime in days
    #     |         |    |  |-------- 3: number of available updates
    #     |         |    |----------- 2: OS major version (string). Must match with OS model data
    #     |         |---------------- 1: OS distribution. Must match with OS model data
    #     |-------------------------- 0: hostname (non-null required)

    # TODO: import multiple

    # TODO: cache hosts?

    os_unknown, _ = Os.objects.get_or_create(distribution='Unknown', version='1')
    # and retrieve all servers
    servers_all   = [[s.hostname, s] for s in Server.objects.select_related('os').all()]

    # month and day on two digits
    month = "%02d" % int(month)
    day   = "%02d" % int(day)

    file_name = "%s-%s-%s.csv" % (year, month, day)
    file_path = path.join(settings.RESULT_DIR, file_name)
    if path.isfile(file_path):
        content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
        f_o = open(file_path, "rt")
        f_r = reader(f_o, delimiter=';')

        # first, delete old status
        ServerStatus.objects.filter(date=date(int(year), int(month), int(day))).delete()

        # loop on server list
        for row in f_r:
            if len(row) > 2:
                row_hostname        = row[0]
                row_os_distribution = row[1]
                row_os_version      = row[2]
                row_updates         = row[3] if row[3] else None
                row_uptime          = row[4] if row[4] else None
                row_auto_updates    = row[5] if row[5] else None

                # get OS
                cache_key_os = "os_%s_%s" % (row_os_distribution, row_os_version)
                cache_ttl    = 1800
                current_os = cache.get(cache_key_os)
                if not current_os:
                    try:
                        current_os = Os.objects.get(distribution=row_os_distribution, version=row_os_version)
                    except Os.DoesNotExist:
                        current_os = os_unknown
                    cache.set(cache_key_os, current_os, cache_ttl)


                # get server if exists
                found_server = False
                for s in servers_all:
                    if s[0] == row_hostname:
                        found_server = True
                        current_s    = s[1]
                        pass
                # if not, create it
                if not found_server:
                    current_s = Server(hostname=row_hostname)

                # set or update the server OS if needed
                if current_s.os != current_os:
                    current_s.os = current_os
                    current_s.save()

                current_st = ServerStatus(
                        date   = date(int(year), int(month), int(day)),
                        server = current_s
                    )
                current_st.updates      = row_updates
                current_st.uptime       = row_uptime
                current_st.auto_updates = row_auto_updates
                try:
                    current_st.save()
                except IntegrityError:
                    content = content + "<br><div class='alert alert-warning' role='alert'><b>Warning:</b> Multiple status for %s</div>" % row_hostname

    else:
        content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path

    return render(request, 'generic.html', {
            'content': content,
        })


## -----------------------------------------------------------------------------
## IMPORT CSV PACKAGES
## Import packages list from CSV
## -----------------------------------------------------------------------------

@login_required
def import_csv_packages(request, year, month, day):
    # Reminder: the imported file must be in this format:
    # myserver.fr;nompackage;version
    #     |         |          |----- 2: version
    #     |         |---------------- 1: package name
    #     |-------------------------- 0: hostname (non-null required)

    # TODO: improve import: don't look for server for each line, but loop csv
    # before to group by host

    # TODO: cache host?

    # retrieve all servers
    servers_all   = [[s.hostname, s] for s in Server.objects.select_related('os').all()]

    # month and day on two digits
    month = "%02d" % int(month)
    day   = "%02d" % int(day)

    file_name = "%s-%s-%s.csv" % (year, month, day)
    file_path = path.join(settings.RESULT_PACKAGES_DIR, file_name)
    if path.isfile(file_path):
        content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
        f_o = open(file_path, "rt")
        f_r = reader(f_o, delimiter=';')

        # first, delete old packages
        PackageStatus.objects.all().delete()

        # loop on server list
        for row in f_r:

            if len(row) == 3:
                row_hostname        = row[0]
                row_packages_name = row[1]
                row_packages_version = row[2]

                # get server if exists
                found_server = False
                for s in servers_all:
                    if s[0] == row_hostname:
                        found_server = True
                        current_s    = s[1]
                        pass

                if found_server:
                    package_status = PackageStatus(
                        server          = current_s,
                        package_name    = row_packages_name,
                        package_version = row_packages_version

                    )

                    package_status.save()

    else:
        content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path

    return render(request, 'generic.html', {
            'content': content,
        })


## -----------------------------------------------------------------------------
## IMPORT UPDATE GROUPS AND TEAMS
## Updates host membership from racktables
## -----------------------------------------------------------------------------

@login_required
def update_groups(request):
    teams = Team.objects.all()
    content = ""
    for team in teams:
        racktables_url = "https://racktables.smile-hosting.fr/ws/platform?team=" + team.color
        # Get all servers
        servers = []
        groups = yaml_load(requests_get(racktables_url).content)
        if groups is None:
            content += "<div class='alert alert-danger' role='alert'>Error: could not retrive groups from racktables for team {}.</div>".format(team.color)
        else:
            content += "<b>Processing team {}</b><br>".format(team.color)
            for group_dict in groups:
                group_name = list(group_dict.keys())[0]
                group, new = Group.objects.get_or_create(name=group_name)
                if new:
                    content += "<span class='text-success'>group {} created</span><br>".format(group_name)
                else:
                    content += "<span class='text-primary'>group {} exists already</span><br>".format(group_name)

                try:
                    group_servers = yaml_load(requests_get("https://racktables.smile-hosting.fr/ws/platform/{}".format(group_name)).content)
                except:
                    content = content + "<div class='alert alert-danger' role='alert'>Parsing error for group {}.</div>".format(group_name)
                    continue

                if group_servers is not None:
                    for server in group_servers:
                        if 'FQDN' in server.keys():
                            if 'Status' not in server.keys() or server['Status'] == 'active':
                                hostname = server['FQDN']
                                content += "&nbsp;&nbsp;&nbsp;&nbsp; updating host {}: ".format(hostname)
                                try:
                                    server = Server.objects.get(hostname=hostname)
                                except Server.DoesNotExist:
                                    server = None
                                if server:
                                    # group management
                                    if not server.group:
                                        server.group = group
                                        server.save()
                                        content += "<span class='text-success'>linked to group</span> - "
                                    elif server.group == group:
                                        content += "<span class='text-info'>group already set</span> - "
                                    else:
                                        server.group = group
                                        server.save()
                                        content += "<span class='text-success'>group updated</span> - "

                                    # team management
                                    if not server.team:
                                        server.team = team
                                        server.save()
                                        content += "<span class='text-success'>linked to team</span><br>"
                                    elif server.team == team:
                                        content += "<span class='text-info'>team already set</span><br>"
                                    else:
                                        server.team = team
                                        server.save()
                                        content += "<span class='text-success'>team updated</span><br>"
                                else:
                                    content += "<span class='text-warning'>server not found in database</span><br>"

    return render(request, 'generic.html', {
            'content': content,
        })


## -----------------------------------------------------------------------------
## MANAGE
## Manage statuses and files
## -----------------------------------------------------------------------------

@login_required
def manage(request):
    # existing dates?
    stat_dates = sorted(list(set([s[0] for s in ServerStatus.objects.values_list('date')])))

    # available files?
    files_dates = []
    for file_name in sorted(listdir(settings.RESULT_DIR)):
        # is it xxxx-xx-xx.csv?
        d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
        # group(1) = 2019, group(2) = 09, group(3) = 20
        if d and stat(path.join(settings.RESULT_DIR, file_name)).st_size > 0:
            files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))

    all_dates = sorted(list(set(stat_dates + files_dates)))

    all_dates_status = []
    for d in all_dates:
        if d in stat_dates:
            all_dates_status.append([d, True])
        else:
            all_dates_status.append([d, False])

    return render(request, 'manage.html', {
            'all_dates_status': reversed(all_dates_status),
        })


## -----------------------------------------------------------------------------
## MANAGE PACKAGES
## Manage packages and files
## -----------------------------------------------------------------------------

@login_required
def manage_packages(request):
    # available files?
    files_dates = []
    for file_name in sorted(listdir(settings.RESULT_PACKAGES_DIR)):
        # is it xxxx-xx-xx.csv?
        d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
        # group(1) = 2019, group(2) = 09, group(3) = 20
        if d and stat(path.join(settings.RESULT_PACKAGES_DIR, file_name)).st_size > 0:
            files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))

    all_dates = sorted(files_dates)

    return render(request, 'manage-packages.html', {
            'all_dates': reversed(all_dates),
        })