# python
from os                         import listdir, path, stat
from re                         import match
from datetime                   import date, datetime, timedelta
from calendar                   import monthrange
from csv                        import reader
from statistics                 import mean, median
from yaml                       import load as yaml_load
from requests                   import get as requests_get
# django
from django.shortcuts               import render, get_object_or_404, redirect
from django.http                    import HttpResponse, HttpResponseNotFound
from django.conf                    import settings
from django.core.exceptions         import ObjectDoesNotExist
from django.core.cache              import cache
from django.db                      import IntegrityError
from django.contrib.auth.decorators import login_required
# project
from .models                    import Os, Server, ServerStatus, PackageStatus, Document_Servers, Document_Packages
from .forms                         import DocumentForm


## -----------------------------------------------------------------------------
## INDEX
## Home page
## -----------------------------------------------------------------------------

@login_required
def index(request):
    return render(request, 'index.html', {})


## -----------------------------------------------------------------------------
## SERVER LIST
## Show servers, their status, number of updates...
## -----------------------------------------------------------------------------

@login_required
def server_list(request, year=None, month=None, day=None):
    # TODO: use date.today?
    now = datetime.now()
    if not year or not month or not day:
        year  = now.year
        month = now.month
        day   = now.day
    year  = int(year)
    month = int(month)
    day   = int(day)

    # check if this is a day
    try:
        current_date = date(year, month, day)
    except ValueError:
        return HttpResponseNotFound('Page not found')

    # retrieve the date of the fist results before and after current date
    previous_result_date = None
    previous_result = ServerStatus.objects.filter(date__lt=current_date).order_by('date').last()
    if previous_result is not None:
        previous_result_date = previous_result.date
    next_result_date = None
    next_result = ServerStatus.objects.filter(date__gt=current_date).order_by('date').first()
    if next_result is not None:
        next_result_date = next_result.date

    results_date = current_date
    # status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname')
    status_list = ServerStatus.objects.filter(date=current_date).order_by('server__hostname').select_related('server', 'server__os')
    if previous_result and not status_list:
        status_list = ServerStatus.objects.filter(date=previous_result_date).order_by('server__hostname').select_related('server', 'server__os')
        results_date = previous_result_date

    if not status_list:
        return render(request, 'generic.html', {
                'content': '<p>No results found.</p>',
            })

    return render(request, 'server-list.html',
                  {
                      'status_list':          status_list,
                      'results_date':         results_date,
                      'previous_result_date': previous_result_date,
                      'next_result_date':     next_result_date,
                  })


## -----------------------------------------------------------------------------
## PACKAGES LIST
## Show package list
## -----------------------------------------------------------------------------

@login_required
def packages_list(request):
    packages_list_all = [p.package_name for p in PackageStatus.objects.all()]
    packages_list     = []

    for p in set(packages_list_all):
        packages_list.append([p, packages_list_all.count(p)])

    return render(request, 'packages-list.html', { 'packages_list': packages_list, })


## -----------------------------------------------------------------------------
## SHOW PACKAGE
## Show servers and versions of a package
## -----------------------------------------------------------------------------

@login_required
def packages(request, package=None , hostname=None):
    # what do we want?
    if package is not None:
        # show all servers that have this package
        packages = PackageStatus.objects.filter(package_name=package)
    else:
        if hostname is not None:
            # show package for one host
            packages = PackageStatus.objects.filter(server__hostname=hostname)
        else:
            # show all packages for all hosts
            packages = PackageStatus.objects.all()

    return render(request, 'packages.html', { 'packages': packages, })


## -----------------------------------------------------------------------------
## OS STATISTICS
## Show percentage for each distrib / version, and a nice chart
## -----------------------------------------------------------------------------

@login_required
def os_statistics(request):
    # get last date for which we have stats
    last_date       = ServerStatus.objects.all().order_by('date').last().date
    current_st_list = ServerStatus.objects.filter(date=last_date)
    nb_servers      = len(current_st_list)
    os_list         = Os.objects.all()
    os_stat         = []
    js_data         = ""

    if last_date is not None:
        # count how many server we have for each distribution
        for os in os_list:
            nb = len(current_st_list.filter(server__os=os))

            if nb > 0:
                os_stat.append([str(os), nb, nb * 100 / nb_servers])
                # for the chart
                js_data = js_data + """
                    {
                        label: "%s",
                        value:  %d
                    },""" % (str(os), nb)

    return render(request, 'os-statistics.html', {
            'js_data':   js_data,
            'os_stat':   os_stat,
            'last_date': last_date,
        })


## -----------------------------------------------------------------------------
## HISTORY
## Show graphs for a month: updates or uptime or os-eol
## -----------------------------------------------------------------------------

@login_required
def history(request, obj, year=False, month=False):
    if not year:
        year = datetime.now().year
    if not month:
        month = datetime.now().month

    if int(month) == int(datetime.now().month):
        current_month = True
    else:
        current_month = False

    year  = int(year)
    month = int(month)

    pc_js_data   = ""
    mean_js_data = ""

    if current_month:
        num_days = 30
        days     = [datetime.now() - timedelta(days=n) for n in range(num_days, -1, -1)]
    else:
        num_days = monthrange(year,month)[1]
        days     = [date(year, month, day) for day in range(1, num_days+1)]

    if obj == 'updates':
        title     = "Updates - " + str(month) + "/" + str(year)
        js_labels = "'Up-to-date (%)', 'Need update (%)', 'Outdated (%)', 'Unknown (%)'"
        title1    = "Updates status repartition (1 month)"
        title2    = "Updates statistics repartition (1 month)"
        fa_icon   = "refresh"
    elif obj == 'uptime':
        title     = "Uptime - " + str(month) + "/" + str(year)
        js_labels = "'Rebooted recently (%)', 'Need a reboot (%)', 'Never rebooted (%)', 'Unknown (%)'"
        title1    = "Uptime status repartition (1 month)"
        title2    = "Uptime statistics repartition (1 month)"
        fa_icon   = "refresh"
    elif obj == 'os':
        title     = "OS status - " + str(month) + "/" + str(year)
        js_labels = "'Maintained (%)', 'End of support soon (%)', 'Out of support (%)', 'Unknown (%)'"
        title1    = "OS status repartition (1 month)"
        title2    = ""
        fa_icon   = "refresh"
    else:
        return HttpResponseNotFound('Page not found')

    # to process mean
    pc_ok_list = []

    # legends
    if obj == 'updates':
        legend1 = ["#updates < 20", "20 < #updates < 100", "100 < #updates", "unknown number of updates"]
        legend2 = ["mean #updates", "median #updates"]
    elif obj == 'uptime':
        legend1 = ["uptime < 100 days", "100 < uptime < 365 days", "365 days < uptime", "unknown uptime"]
        legend2 = ["mean uptime", "median uptime"]
    elif obj == 'os':
        legend1 = ["OS maintained for more than 1 year", "OS unmaintained in less than 1 year", "OS unmaintained", "unknown OS"]
        legend2 = []

    # generate data (javascript) for each day
    some_results = False
    for day in days:
        status = ServerStatus.objects.filter(date=day)
        if status.count() != 0:
            some_results = True
            nb_tot = status.count() + 1
            if obj == 'updates':
                # pourcentage
                pc_ok   = len([s for s in status if s.updates_status()==1]) * 100 / nb_tot
                pc_warn = len([s for s in status if s.updates_status()==2]) * 100 / nb_tot
                pc_crit = len([s for s in status if s.updates_status()==3]) * 100 / nb_tot
                pc_unk  = len([s for s in status if s.updates_status()==0]) * 100 / nb_tot
                # mean / median
                updates_list = [upd for upd in status.values_list('updates', flat=True) if upd]
                mean_val     = mean(updates_list or [0])
                median_val   = median(updates_list or [0])
                # to process mean
                pc_ok_list = pc_ok_list + [pc_ok]
            elif obj == 'uptime':
                # pourcentage
                pc_ok   = len([s for s in status if s.uptime_status()==1]) * 100 / nb_tot
                pc_warn = len([s for s in status if s.uptime_status()==2]) * 100 / nb_tot
                pc_crit = len([s for s in status if s.uptime_status()==3]) * 100 / nb_tot
                pc_unk  = len([s for s in status if s.uptime_status()==0]) * 100 / nb_tot
                # mean / median
                uptime_list = [upd for upd in status.values_list('uptime', flat=True) if upd]
                mean_val    = mean(uptime_list or [0])
                median_val  = median(uptime_list or [0])
            elif obj == 'os':
                # pourcentage
                pc_ok   = len([s for s in status if s.os_status()==1]) * 100 / nb_tot
                pc_warn = len([s for s in status if s.os_status()==2]) * 100 / nb_tot
                pc_crit = len([s for s in status if s.os_status()==3]) * 100 / nb_tot
                pc_unk  = len([s for s in status if s.os_status()==0]) * 100 / nb_tot

            pc_js_data = pc_js_data + """
            {
                period:  '%s',
                ok:      %.2f,
                warn:    %.2f,
                crit:    %.2f,
                unknown: %.2f
            },""" % (day, pc_ok, pc_warn, pc_crit, pc_unk)

            if obj == 'os':
                mean_js_data = False
            else:
                mean_js_data = mean_js_data + """
                {
                    period:  '%s',
                    mean:    %d,
                    median:  %d
                },""" % (day, mean_val, median_val)

    mean_pc_ok = None
    if obj == 'updates':
        # pc_ok_list may be empty
        if len(pc_ok_list) > 0:
            mean_pc_ok = mean(pc_ok_list)
        else:
            mean_pc_ok = 0

    return render(request, 'history.html',
                  {
                      'some_results': some_results,
                      'pc_js_data':   pc_js_data,
                      'mean_js_data': mean_js_data,
                      'title':        title,
                      'title1':       title1,
                      'title2':       title2,
                      'legend1':      legend1,
                      'legend2':      legend2,
                      'js_labels':    js_labels,
                      'fa_icon':      'refresh',
                      'obj':          obj,
                      'year':         year,
                      'month':        "%02d" % month,
                      'months':       ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'],
                      'years':        [datetime.now().year-1, datetime.now().year],
                      'mean_pc_ok':   mean_pc_ok,
                  })


## -----------------------------------------------------------------------------
## PURGE ALL
## Purge everything in database. Dangerous!
## -----------------------------------------------------------------------------

@login_required
def purge_all(request):
    Server.objects.all().delete()
    ServerStatus.objects.all().delete()
    Os.objects.all().delete()
    PackageStatus.objects.all().delete()

    content = "<div class='alert alert-danger' role='alert'>Everything has been purged.</div>"

    return render(request, 'generic.html', {content: content})


## -----------------------------------------------------------------------------
## PURGE
## Purge statuses for one specific date
## -----------------------------------------------------------------------------

@login_required
def purge_statuses_by_date(request, year, month, day):
    status = ServerStatus.objects.filter(date=date(int(year), int(month), int(day)))
    if status:
        content = "<div class='alert alert-success' role='alert'>Status for %s-%s-%s have been removed.</div>" % (year, month, day)
        status.delete()
    else:
        content = "<div class='alert alert-danger' role='alert'>No server status for this date: %s-%s-%s.</div>" % (year, month, day)

    return render(request, 'generic.html', { 'content': content, })


## -----------------------------------------------------------------------------
## PURGE PACKAGES
## Purge all packages
## -----------------------------------------------------------------------------

@login_required
def purge_packages(request):
    Server.objects.all().delete()
    PackageStatus.objects.all().delete()
    Os.objects.all().delete()

    content = "<div class='alert alert-danger' role='alert'>Packages have been purged.</div>"

    return render(request, 'generic.html', {content: content})


## -----------------------------------------------------------------------------
## IMPORT CSV
## Import statuses list from CSV
## -----------------------------------------------------------------------------

@login_required
def import_csv(request, year, month, day):
    # Reminder: the imported file must be in this format:
    # myserver.fr;Ubuntu;16;28;12
    #     |         |    |  |  |----- 4: uptime in days
    #     |         |    |  |-------- 3: number of available updates
    #     |         |    |----------- 2: OS major version (string). Must match with OS model data
    #     |         |---------------- 1: OS distribution. Must match with OS model data
    #     |-------------------------- 0: hostname (non-null required)

    # TODO: import multiple

    # TODO: cache hosts?

    os_unknown, _ = Os.objects.get_or_create(distribution='Unknown', version='1')
    # and retrieve all servers
    servers_all   = [[s.hostname, s] for s in Server.objects.select_related('os').all()]

    # month and day on two digits
    month = "%02d" % int(month)
    day   = "%02d" % int(day)

    file_name = "%s-%s-%s.csv" % (year, month, day)
    file_path = path.join(settings.RESULT_DIR, file_name)
    if path.isfile(file_path):
        content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
        f_o = open(file_path, "rt")
        f_r = reader(f_o, delimiter=';')

        # first, delete old status
        ServerStatus.objects.filter(date=date(int(year), int(month), int(day))).delete()

        # loop on server list
        for row in f_r:
            if len(row) > 2:
                row_hostname        = row[0]
                row_os_distribution = row[1]
                row_os_version      = row[2]
                row_updates         = row[3] if row[3] else None
                row_uptime          = row[4] if row[4] else None

                # get OS
                cache_key_os = "os_%s_%s" % (row_os_distribution, row_os_version)
                cache_ttl    = 1800
                current_os = cache.get(cache_key_os)
                if not current_os:
                    try:
                        current_os = Os.objects.get(distribution=row_os_distribution, version=row_os_version)
                    except Os.DoesNotExist:
                        current_os = os_unknown
                    cache.set(cache_key_os, current_os, cache_ttl)


                # get server if exists
                found_server = False
                for s in servers_all:
                    if s[0] == row_hostname:
                        found_server = True
                        current_s    = s[1]
                        pass
                # if not, create it
                if not found_server:
                    current_s = Server(hostname=row_hostname)

                # set or update the server OS if needed
                if current_s.os != current_os:
                    current_s.os = current_os
                    current_s.save()

                current_st = ServerStatus(
                        date   = date(int(year), int(month), int(day)),
                        server = current_s
                    )
                current_st.updates      = row_updates
                current_st.uptime       = row_uptime
                try:
                    current_st.save()
                except IntegrityError:
                    content = content + "<br><div class='alert alert-warning' role='alert'><b>Warning:</b> Multiple status for %s</div>" % row_hostname

    else:
        content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path

    return render(request, 'generic.html', {
            'content': content,
        })


## -----------------------------------------------------------------------------
## IMPORT CSV PACKAGES
## Import packages list from CSV
## -----------------------------------------------------------------------------

@login_required
def import_csv_packages(request, year, month, day):
    # Reminder: the imported file must be in this format:
    # myserver.fr;nompackage;version
    #     |         |          |----- 2: version
    #     |         |---------------- 1: package name
    #     |-------------------------- 0: hostname (non-null required)

    # TODO: improve import: don't look for server for each line, but loop csv
    # before to group by host

    # TODO: cache host?

    # retrieve all servers
    servers_all   = [[s.hostname, s] for s in Server.objects.select_related('os').all()]

    # month and day on two digits
    month = "%02d" % int(month)
    day   = "%02d" % int(day)

    file_name = "%s-%s-%s.csv" % (year, month, day)
    file_path = path.join(settings.RESULT_PACKAGES_DIR, file_name)
    if path.isfile(file_path):
        content = "<br><div class='alert alert-success' role='alert'>Database have been updated for %s %s %s</div>" % (year, month, day)
        f_o = open(file_path, "rt")
        f_r = reader(f_o, delimiter=';')

        # first, delete old packages
        PackageStatus.objects.all().delete()

        # loop on server list
        for row in f_r:

            if len(row) == 3:
                row_hostname        = row[0]
                row_packages_name = row[1]
                row_packages_version = row[2]

                # get server if exists
                found_server = False
                for s in servers_all:
                    if s[0] == row_hostname:
                        found_server = True
                        current_s    = s[1]
                        pass

                if found_server:
                    package_status = PackageStatus(
                        server          = current_s,
                        package_name    = row_packages_name,
                        package_version = row_packages_version

                    )

                    package_status.save()

    else:
        content = "<div class='alert alert-danger' role='alert'>File not found: %s</div>" % file_path

    return render(request, 'generic.html', {
            'content': content,
        })

## -----------------------------------------------------------------------------
## MANAGE
## Manage statuses and files
## -----------------------------------------------------------------------------

@login_required
def manage(request):
    # existing dates?
    stat_dates = sorted(list(set([s[0] for s in ServerStatus.objects.values_list('date')])))

    # available files?
    files_dates = []
    for file_name in sorted(listdir(settings.RESULT_DIR)):
        # is it xxxx-xx-xx.csv?
        d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
        # group(1) = 2019, group(2) = 09, group(3) = 20
        if d and stat(path.join(settings.RESULT_DIR, file_name)).st_size > 0:
            files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))

    all_dates = sorted(list(set(stat_dates + files_dates)))

    all_dates_status = []
    for d in all_dates:
        if d in stat_dates:
            all_dates_status.append([d, True])
        else:
            all_dates_status.append([d, False])

    return render(request, 'manage.html', {
            'all_dates_status': reversed(all_dates_status),
        })


## -----------------------------------------------------------------------------
## MANAGE PACKAGES
## Manage packages and files
## -----------------------------------------------------------------------------

@login_required
def manage_packages(request):
    # available files?
    files_dates = []
    for file_name in sorted(listdir(settings.RESULT_PACKAGES_DIR)):
        # is it xxxx-xx-xx.csv?
        d = match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})\.csv$", file_name)
        # group(1) = 2019, group(2) = 09, group(3) = 20
        if d and stat(path.join(settings.RESULT_PACKAGES_DIR, file_name)).st_size > 0:
            files_dates.append(date(int(d.group(1)), int(d.group(2)), int(d.group(3))))

    all_dates = sorted(files_dates)

    return render(request, 'manage-packages.html', {
            'all_dates': reversed(all_dates),
        })

## -----------------------------------------------------------------------------
## UPLOAD FILE
## Upload csv file for servers informations
## -----------------------------------------------------------------------------

@login_required
def upload_csv_results(request):
    message = 'File must be name YYYY-MM-DD.csv'
    # Handle file upload
    if request.method == 'POST':
        form = DocumentForm(request.POST, request.FILES)
        if form.is_valid():
            newdoc = Document_Servers(docfile=request.FILES['docfile'])
            newdoc.save()

            # Redirect to the document list after POST
            return redirect('manage')
        else:
            message = 'The form is not valid. Fix the following error:'
    else:
        form = DocumentForm()  # An empty, unbound form

    # Load documents for the list page
    documents = Document_Servers.objects.all()

    # Render list page with the documents and the form
    context = {'documents': documents, 'form': form, 'message': message}
    return render(request, 'manage.html', context)

## -----------------------------------------------------------------------------
## UPLOAD FILE
## Upload csv file for packages informations
## -----------------------------------------------------------------------------

@login_required
def upload_csv_results_packages(request):
    message = 'File must be name YYYY-MM-DD.csv'
    # Handle file upload
    if request.method == 'POST':
        form = DocumentForm(request.POST, request.FILES)
        if form.is_valid():
            newdoc = Document_Packages(docfile=request.FILES['docfile'])
            newdoc.save()

            # Redirect to the document list after POST
            return redirect('manage-packages')
        else:
            message = 'The form is not valid. Fix the following error:'
    else:
        form = DocumentForm()  # An empty, unbound form

    # Load documents for the list page
    documents = Document_Packages.objects.all()

    # Render list page with the documents and the form
    context = {'documents': documents, 'form': form, 'message': message}
    return render(request, 'manage-packages.html', context)