SITools2

__version__ = "0.1"

__license__ = "GPL"

__author__ = "Marc NICOLAS"

__credit__ = "Marc NICOLAS"

__maintainer__ = "Marc NICOLAS"

__email__ = "marc.nicolas@ias.u-psud.fr"

import sys, os

try:

    import copy

except:

    sys.exit("Import failed in module libStatSitools :\n\tcopy module is required")

try:

    import pygeoip

except:

    sys.exit("Import failed in module libStatSitools :\n\tpygeoip module is required")

try:

    from datetime import datetime, timedelta

except:

    sys.exit("Import failed in module libStatSitools :\n\tdatetime module is required")

try:

    import os

except:

    sys.exit("Import failed in module libStatSitools :\n\tos module is required")

try:

    import string

except:

    sys.exit("Import failed in module libStatSitools :\n\tstring module is required")

try:

    import time

except:

    sys.exit("Import failed in module libStatSitools :\n\ttime module is required")

try:

    import copy

except:

    sys.exit("Import failed in module libStatSitools :\n\tcopy module is required")

try:

    from libChart4StatSitools2 import *

except:

    sys.exit("Import failed in module lancementStat :\n\tlibChart4StatSitools2 module is required")

try:

    from libStatSitoolsConstante import *

except:

    sys.exit("Import failed in module lancementStat :\n\tlibStatSitoolsConstante module is required")

try:

    import re

except:

    sys.exit("Import failed in module lancementStat :\n\tre module is required")

try:

    import operator

except:

    sys.exit("Import failed in module lancementStat :\n\toperator module is required")

try:

    # PATH FOR PROD

    #gi = pygeoip.GeoIP("/usr/local/sitools2_webstat/webstat/GeoIP.dat")

    # PATH FOR DEV

    gi = pygeoip.GeoIP("/home/marc/Projet/idoc_Maison/webstat/GeoIP.dat")

except:

    sys.exit("Import of GeoIP.dat failed. Verify that the GeoIP file exists !")

time_format = "%Y-%m-%d %H:%M:%S"

time_format_for_by_month = "%Y-%m-%d"

def reduce_dico(dico, max_per_cent):

    dico2 = refactor_dico(dico, max_per_cent)

    return dico2

def stat_log(application, cond_app):

    try:

        dico_cste = defCst(application)

        dico_project = dico_cste["dicoProject"]

        list_path_logs = dico_cste["listPathLogs"]

        start_date = cond_app["start_date"]

        end_date = cond_app["end_date"]

        #if cond_app["is_access_access_by_country"]:

        reg_infos_url = re.compile(ur'\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}')

        #************************* VARIABLE POUR LES STAT D'ACCES *****************************************

        dico_access_ip = {}

        dico_access_by_country = {}

        dico_access_by_month = {}

        nb_access_ip = 0

        dico_vo_ip = {}

        dico_vo_by_country = {}

        nb_vo_ip = 0

        #************************* VARIABLE POUR LES STAT DE DOWNLOAD *****************************************

        vol_download = 0

        nb_download = 0

        total_down_by_months = {}

        total_down_during_period = 0

        total_down_per_country = {}

        total_down_per_project = {}

        total_down_per_user = {}

        total_down_per_ip = {}

        for pathLog in list_path_logs:

            listarbo = os.listdir(pathLog)

            print "PathLog : ",pathLog

        for fileToOpen in listarbo:

            pathfile = pathLog + "/" + fileToOpen

            if os.path.isfile(pathfile) and os.path.basename(fileToOpen).split(".")[1] == "log" and \

                os.path.getsize(pathfile) != 0  and "sitools-log-service" in fileToOpen:

                print "Traitement de ",pathfile

                print  "taille : ",os.path.getsize(pathfile)

                fileopen = open(pathfile, "r")

                lines = fileopen.readlines()

                for line in lines:

                    infos = line.split("\t")

                    my_time = datetime.strptime(infos[0].split(" ")[0][:len(infos[0].split(" ")[0]) - 4].replace("T", " ").split(" ")[0], time_format_for_by_month).date()

                    if len(infos) > 14 and infos[2] not in dico_cste["listIpToExclude"] and dico_cste["patternBot"] not in infos[14] and dico_cste["patternYahoo"] not in infos[14] \

                        and not any(s in infos[14] for s in dico_cste["patternBaidu"]):

                        if start_date < my_time < end_date:

                        #----------------------------------------------------------------------------------------------

                        #  **************************** STATISTIQUE SUR LES DOWNLOAD *******************************

                        #-----------------------------------------------------------------------------------------------

                            if len(infos) == 16 and infos[6] == dico_cste["pattern_http_method"] and infos[9] == dico_cste["pattern_code_retour"] and any(s in infos[7] for s in dico_cste["pattern_download"]) \

                                and not any(s in infos[7] for s in dico_cste["sia"]) and not any(s in infos[7] for s in dico_cste["CutFits"]) and gi.country_code_by_addr(infos[2]) != "CN" :

                                    nb_download += 1

                                    try:

                                        vol_download += float(infos[10])

                                    except:

                                        pass

                                    ################################ DOWNLOAD BY PROJECT ##########################

                                    if cond_app["is_down_stat_by_project"]:

                                        if len(infos) == 16:

                                            project_tmp = infos[15].split("/")

                                            if project_tmp[0] == "-\n":

                                                project = infos[7].split("/")[8]

                                            elif len(project_tmp) == 7:

                                                project = project_tmp[5]

                                            else:

                                                project = "others"

                                            if project.lower() not in dico_project.keys():

                                                project = "others"

                                        else:

                                            if infos[15] != "-\n":

                                                project = "others"

                                            else:

                                                if len(infos[7].split("/")) < 8:

                                                    project = "others"

                                                else:

                                                    project_tmp2 = infos[7].split("/")[8]

                                                    #print "project_tmp2 : ",project_tmp2

                                                    if project_tmp2.lower() not in dico_project.keys():

                                                        project = "others"

                                                    else:

                                                        project = project_tmp2

                                        try:

                                            if dico_project[project.lower()] not in total_down_per_project.keys():

                                                total_down_per_project[dico_project[project.lower()]] = float(infos[10])

                                            else:

                                                total_down_per_project[dico_project[project.lower()]] += float(infos[10])

                                        except:

                                            pass

                                    ############################### DOWNLOAD BY MONTH ###############################

                                    if cond_app["is_down_stat_by_months"]:

                                            try:

                                                if (my_time.month, my_time.year) not in total_down_by_months.keys():

                                                    total_down_by_months[my_time.month, my_time.year] = float(infos[10])

                                                    total_down_during_period += float(infos[10])

                                                else:

                                                    total_down_by_months[my_time.month, my_time.year] += float(infos[10])

                                                    total_down_during_period += float(infos[10])

                                            except:

                                                pass

                                    ############################# DOWNLOAD BY COUNTRY ##################################

                                    if cond_app["is_down_stat_by_country"]:

                                        try:

                                            if gi.country_code_by_addr(infos[2]) not in total_down_per_country.keys():

                                                total_down_per_country[gi.country_code_by_addr(infos[2])] = float(infos[10])

                                            else:

                                                total_down_per_country[gi.country_code_by_addr(infos[2])] += float(infos[10])

                                        except:

                                            pass

                                    ######################### DOWNLOAD BY IP ##########################################

                                    if cond_app["is_down_stat_by_ip"]:

                                        try:

                                            if infos[2] not in total_down_per_ip.keys():

                                                total_down_per_ip[infos[2]] = float(infos[10])

                                            else:

                                                total_down_per_ip[infos[2]] += float(infos[10])

                                        except:

                                            pass

                                    ######################### DOWNLOAD BY USER #######################################

                                    if cond_app["is_down_stat_by_user"]:

                                        pass

                            # AJOUT D'UNE CONDITION SUR 127.0.0.1 POUR EXCLURE HOBBIT PENDANT UNE PERIODE OU ON AVAIT PLUS ACCES AU "VRAI" IP

                            # --------------------------------------------------------------------------------------

                            # ***************************************************   STATISTIQUES SUR LES ACCESS ****

                            if not infos[2] == "127.0.0.1" and re.match(reg_infos_url, infos[2]) is not None and gi.country_code_by_addr(infos[2]) != "CN":

                                #*********************************  ACCESS By IP ******************************************

                                if cond_app["is_access_access_by_ip"]:

                                    if dico_cste["patternAccessToSeek"] == infos[7]:# and gi.country_code_by_addr(infos[2]) != "CN":

                                        # incrementing of number access

                                        nb_access_ip += 1

                                        if infos[2] not in dico_access_ip.keys():

                                            dico_access_ip[infos[2]] = 1

                                        else:

                                            dico_access_ip[infos[2]] += 1

                                #*********************************  ACCESS By COUNTRY *************************************

                                if cond_app["is_access_access_by_country"]:

                                    if dico_cste["patternAccessToSeek"] == infos[7]:# and gi.country_code_by_addr(infos[2]) != "CN":

                                        if re.match(reg_infos_url, infos[2]) is not None:

                                            if gi.country_name_by_name(infos[2]) not in dico_access_by_country.keys():

                                                dico_access_by_country[gi.country_name_by_name(infos[2])] = 1

                                            else:

                                                dico_access_by_country[gi.country_name_by_name(infos[2])] += 1

                                #*********************************  ACCESS By MONTH ***************************************

                                if cond_app["is_access_access_by_months"]:

                                    if dico_cste["patternAccessToSeek"] == infos[7]:# and gi.country_code_by_addr(infos[2]) != "CN":

                                        if (my_time.month, my_time.year) not in dico_access_by_month.keys():

                                            dico_access_by_month[my_time.month, my_time.year] = 1

                                        else:

                                            dico_access_by_month[my_time.month, my_time.year] += 1

                                #*********************************  ACCESS VO By IP ****************************************

                            '''

                            if dico_cste["patternVoConeSearchToSeek"] == infos[7] and \

                                            dico_cste["patternVoConeSearchToSeek"] != "":

                                # incrementing of number access

                                nb_vo_ip += 1

                                if infos[2] not in dico_vo_ip.keys():

                                    dico_vo_ip[infos[2]] = 1

                                else:

                                    dico_vo_ip[infos[2]] += 1

                                if gi.country_name_by_name(infos[2]) not in dico_vo_by_country.keys():

                                    dico_vo_by_country[gi.country_name_by_name(infos[2])] = 1

                                else:

                                    dico_vo_by_country[gi.country_name_by_name(infos[2])] += 1

                            '''

                    # Closing the log file

                    fileopen.close()

        # Creation des graphs :

        date_title_chart = ""

        if cond_app["period_date"]:

            date_title_chart = " entre le "+str(start_date)+" et le "+str(end_date)

        if cond_app["is_access_access_by_ip"]:

            dico_tmp_access_ip = reduce_dico(dico_access_ip,  dico_cste["dico_max_per_cent"]["is_access_access_by_ip"])

            access_ip_chart_svg_png = svg_chart(dico_tmp_access_ip, "Acces par IP"+str(date_title_chart))

        else:

            access_ip_chart_svg_png = None

        if cond_app["is_access_access_by_country"]:

            dico_tmp_access_country = reduce_dico(dico_access_by_country,

                                                  dico_cste["dico_max_per_cent"]["is_access_access_by_country"])

            access_country_chart_svg_png = svg_chart(dico_tmp_access_country, "Acces par pays"+str(date_title_chart))

        else:

            access_country_chart_svg_png = None

        if cond_app["is_access_access_by_months"]:

            access_month_chart_svg_png = svg_bar_chart(dico_access_by_month, start_date, end_date, "Acces par mois"+str(date_title_chart))

        else:

            access_month_chart_svg_png = None

        if cond_app["is_down_stat_by_project"]:

            total_down_per_project_chart = svg_chart(reduce_dico(total_down_per_project,

                                                     dico_cste["dico_max_per_cent"]["is_down_stat_by_project"]),

                                                     "Volume de telechargement par projet"+str(date_title_chart))

        else:

            total_down_per_project_chart = None

        if cond_app["is_down_stat_by_ip"]:

            total_down_per_ip_chart = svg_chart(reduce_dico(total_down_per_ip,

                                                dico_cste["dico_max_per_cent"]["is_down_stat_by_ip"]),

                                                "Volume de telechargement par IP"+str(date_title_chart))

        else:

            total_down_per_ip_chart = None

        if cond_app["is_down_stat_by_months"]:

            total_down_per_month_chart = svg_bar_chart(total_down_by_months, start_date, end_date,"Volume de telechargement par mois"+str(date_title_chart))

        else:

            total_down_per_month_chart = None

        if cond_app["is_down_stat_by_country"]:

            total_down_per_country_chart = svg_chart(reduce_dico(total_down_per_country,

                                                     dico_cste["dico_max_per_cent"]["is_down_stat_by_country"]),

                                                     "Volume de telechargement par Pays"+str(date_title_chart))

        else:

            total_down_per_country_chart = None

        return {"nb_download": nb_download, "vol_download": format_download_vol(vol_download), "nb_access_ip":

                nb_access_ip, "access_ip_chart_svg_png": access_ip_chart_svg_png, "access_country_chart_svg_png":

                access_country_chart_svg_png,"access_month_chart_svg_png": access_month_chart_svg_png,

                "total_down_per_project_chart": total_down_per_project_chart, "total_down_per_month_chart":

                total_down_per_month_chart, "total_down_per_country_chart": total_down_per_country_chart,

                "total_down_per_ip_chart": total_down_per_ip_chart, "dico_cste": dico_cste}

    except BaseException, e:

        print "Error : ", str(e)

def refactor_dico(dico, max):

    total = float(sum(dico.values()))

    dico2 = {}

    for key in dico:

        #print float(dico[key]/total)

        #print "dico[key] : ", dico[key], "     float(dico[key]/total) : ", float(dico[key])/total

        if float((dico[key])/total) > max:

            dico2[key] = dico[key]

    print dico2

    return dico2

def format_download_vol(vol_down):

    vol_data = ""

    '''

    try:

        if 1024 <= vol_down < 1048576:

            vol_data = str(round(vol_down/1024, 2))+" Ko"

        elif 1048576 <= vol_down < 1073741824:

            vol_data = str(round(vol_down/1048576, 2))+" Mo"

        elif 1073741824 <= vol_down < (1073741824*1024):

            vol_data = str(round(vol_down/1073741824, 2))+" Go"

    except BaseException, e:

        print str(e)

    '''

    try:

        if 1000 <= vol_down < 1000000:

            vol_data = str(round(vol_down/1000, 2))+" Ko"

        elif 1000000 <= vol_down < 1000000000:

            vol_data = str(round(vol_down/1000000, 2))+" Mo"

        elif 1000000000 <= vol_down < (1000000000*1000):

            vol_data = str(round(vol_down/1000000000, 2))+" Go"

    except BaseException, e:

        print str(e)

    return vol_data