Revision 9791e4f3
Added by Marc Nicolas almost 9 years ago
| webstatDev/idoc_Maison/webstat/libStatSitools.py | ||
|---|---|---|
| 5 | 5 |
__maintainer__ = "Marc NICOLAS" |
| 6 | 6 |
__email__ = "marc.nicolas@ias.u-psud.fr" |
| 7 | 7 |
|
| 8 |
import sys, os |
|
| 9 |
|
|
| 10 | 8 |
try: |
| 11 | 9 |
import copy |
| 12 | 10 |
except: |
| ... | ... | |
| 52 | 50 |
except: |
| 53 | 51 |
sys.exit("Import failed in module lancementStat :\n\toperator module is required")
|
| 54 | 52 |
try: |
| 55 |
# PATH FOR PROD
|
|
| 53 |
#PATH FOR PROD |
|
| 56 | 54 |
#gi = pygeoip.GeoIP("/usr/local/sitools2_webstat/webstat/GeoIP.dat")
|
| 57 |
# PATH FOR DEV
|
|
| 58 |
gi = pygeoip.GeoIP("/home/marc/MyDev/sitools-idoc/webstatDev/idoc_Maison/webstat/GeoIP.dat")
|
|
| 55 |
#PATH FOR DEV |
|
| 56 |
gi = pygeoip.GeoIP("/home/marc/Projet/idoc_Maison/webstat/GeoIP.dat")
|
|
| 59 | 57 |
except: |
| 60 | 58 |
sys.exit("Import of GeoIP.dat failed. Verify that the GeoIP file exists !")
|
| 61 | 59 |
|
| ... | ... | |
| 103 | 101 |
|
| 104 | 102 |
for pathLog in list_path_logs: |
| 105 | 103 |
listarbo = os.listdir(pathLog) |
| 106 |
print "PathLog : ",pathLog
|
|
| 104 |
print "PathLog : ", pathLog
|
|
| 107 | 105 |
for fileToOpen in listarbo: |
| 108 | 106 |
pathfile = pathLog + "/" + fileToOpen |
| 109 |
if os.path.isfile(pathfile) and os.path.basename(fileToOpen).split(".")[1] == "log" and os.path.getsize(pathfile) != 0 and "sitools-log-service" in fileToOpen:
|
|
| 110 |
print "Traitement de ",pathfile |
|
| 111 |
print "taille : ",os.path.getsize(pathfile) |
|
| 107 |
if os.path.isfile(pathfile) and os.path.basename(fileToOpen).split(".")[1] == "log" and \
|
|
| 108 |
os.path.getsize(pathfile) != 0 and "sitools-log-service" in fileToOpen: |
|
| 109 |
print "Traitement de ", pathfile |
|
| 110 |
print "taille : ", os.path.getsize(pathfile) |
|
| 112 | 111 |
fileopen = open(pathfile, "r") |
| 113 | 112 |
lines = fileopen.readlines() |
| 114 | 113 |
for line in lines: |
| 115 | 114 |
infos = line.split("\t")
|
| 116 |
my_time = datetime.strptime(infos[0].split(" ")[0][:len(infos[0].split(" ")[0]) - 4].replace("T", " ").split(" ")[0], time_format_for_by_month).date()
|
|
| 115 |
my_time = datetime.strptime(infos[0].split(" ")[0][:len(infos[0].split(" ")[0]) - 4].replace("T"," ").split(" ")[0], time_format_for_by_month).date()
|
|
| 117 | 116 |
if len(infos) > 14 and infos[2] not in dico_cste["listIpToExclude"] and dico_cste["patternBot"] not in infos[14] and dico_cste["patternYahoo"] not in infos[14] \ |
| 118 |
and not any(s in infos[14] for s in dico_cste["patternBaidu"]):
|
|
| 117 |
and not any(s in infos[14] for s in dico_cste["patternBaidu"]):
|
|
| 119 | 118 |
if start_date < my_time < end_date: |
| 120 |
#---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
| 121 |
# ******************************************************************* STATISTIQUE SUR LES DOWNLOAD *****************************************************************
|
|
| 122 |
#-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
| 123 |
if len(infos) == 16 and infos[6] == dico_cste["pattern_http_method"] and infos[9] == dico_cste["pattern_code_retour"] and any(s in infos[7] for s in dico_cste["pattern_download"]) \
|
|
| 119 |
#------------------------------------------------------------------------------------ |
|
| 120 |
# ********************************** STATISTIQUE SUR LES DOWNLOAD ********************
|
|
| 121 |
#---------------------------------------------------------------------------------------
|
|
| 122 |
if len(infos) == 16 and infos[6] == dico_cste["pattern_http_method"] and infos[9] == dico_cste["pattern_code_retour"] and any(s in infos[7] for s in dico_cste["pattern_download"]) \
|
|
| 124 | 123 |
and not any(s in infos[7] for s in dico_cste["sia"]) and not any(s in infos[7] for s in dico_cste["CutFits"]) and gi.country_code_by_addr(infos[2]) != "CN" : |
| 125 | 124 |
nb_download += 1 |
| 126 | 125 |
try: |
| ... | ... | |
| 147 | 146 |
project = "others" |
| 148 | 147 |
else: |
| 149 | 148 |
project_tmp2 = infos[7].split("/")[8]
|
| 150 |
#print "project_tmp2 : ",project_tmp2 |
|
| 151 | 149 |
if project_tmp2.lower() not in dico_project.keys(): |
| 152 | 150 |
project = "others" |
| 153 | 151 |
else: |
| ... | ... | |
| 191 | 189 |
######################### DOWNLOAD BY USER ####################################### |
| 192 | 190 |
if cond_app["is_down_stat_by_user"]: |
| 193 | 191 |
pass |
| 194 |
# AJOUT D'UNE CONDITION SUR 127.0.0.1 POUR EXCLURE HOBBIT PENDANT UNE PERIODE OU ON AVAIT PLUS ACCES AU "VRAI" IP |
|
| 195 |
#--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| 196 |
# *************************************************** STATISTIQUES SUR LES ACCESS ************************************************************* |
|
| 197 |
#------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| 198 |
if not infos[2] == "127.0.0.1" and re.match(reg_infos_url, infos[2]) is not None and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 199 |
#********************************* ACCESS By IP ****************************************** |
|
| 200 |
if cond_app["is_access_access_by_ip"]: |
|
| 201 |
if dico_cste["patternAccessToSeek"] == infos[7]:# and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 202 |
# incrementing of number access |
|
| 203 |
nb_access_ip += 1 |
|
| 204 |
if infos[2] not in dico_access_ip.keys(): |
|
| 205 |
dico_access_ip[infos[2]] = 1 |
|
| 206 |
else: |
|
| 207 |
dico_access_ip[infos[2]] += 1 |
|
| 208 |
#********************************* ACCESS By COUNTRY ************************************* |
|
| 209 |
if cond_app["is_access_access_by_country"]: |
|
| 210 |
if dico_cste["patternAccessToSeek"] == infos[7]:# and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 211 |
if re.match(reg_infos_url, infos[2]) is not None: |
|
| 212 |
if gi.country_name_by_name(infos[2]) not in dico_access_by_country.keys(): |
|
| 213 |
dico_access_by_country[gi.country_name_by_name(infos[2])] = 1 |
|
| 214 |
else: |
|
| 215 |
dico_access_by_country[gi.country_name_by_name(infos[2])] += 1 |
|
| 216 |
#********************************* ACCESS By MONTH *************************************** |
|
| 217 |
if cond_app["is_access_access_by_months"]: |
|
| 218 |
if dico_cste["patternAccessToSeek"] == infos[7]:# and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 219 |
if (my_time.month, my_time.year) not in dico_access_by_month.keys(): |
|
| 220 |
dico_access_by_month[my_time.month, my_time.year] = 1 |
|
| 221 |
else: |
|
| 222 |
dico_access_by_month[my_time.month, my_time.year] += 1 |
|
| 223 |
#********************************* ACCESS VO By IP **************************************** |
|
| 192 |
# AJOUT D'UNE CONDITION SUR 127.0.0.1 POUR EXCLURE HOBBIT\ |
|
| 193 |
# PENDANT UNE PERIODE OU ON AVAIT PLUS ACCES AU "VRAI" IP |
|
| 194 |
#-------------------------------------------------------------------------------------- |
|
| 195 |
# *************************** STATISTIQUES SUR LES ACCESS **************************** |
|
| 196 |
#--------------------------------------------------------------------------------------- |
|
| 197 |
if not infos[2] == "127.0.0.1" and re.match(reg_infos_url, infos[2]) \ |
|
| 198 |
is not None and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 199 |
#********************************* ACCESS By IP ******************************* |
|
| 200 |
if cond_app["is_access_access_by_ip"]: |
|
| 201 |
if dico_cste["patternAccessToSeek"] == infos[7] and \ |
|
| 202 |
gi.country_code_by_addr(infos[2]) != "CN": |
|
| 203 |
# incrementing of number access |
|
| 204 |
nb_access_ip += 1 |
|
| 205 |
if infos[2] not in dico_access_ip.keys(): |
|
| 206 |
dico_access_ip[infos[2]] = 1 |
|
| 207 |
else: |
|
| 208 |
dico_access_ip[infos[2]] += 1 |
|
| 209 |
#********************************* ACCESS By COUNTRY ************************* |
|
| 210 |
if cond_app["is_access_access_by_country"]: |
|
| 211 |
if dico_cste["patternAccessToSeek"] == infos[7]\ |
|
| 212 |
and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 213 |
if re.match(reg_infos_url, infos[2]) is not None: |
|
| 214 |
if gi.country_name_by_name(infos[2]) not in dico_access_by_country.keys(): |
|
| 215 |
dico_access_by_country[gi.country_name_by_name(infos[2])] = 1 |
|
| 216 |
else: |
|
| 217 |
dico_access_by_country[gi.country_name_by_name(infos[2])] += 1 |
|
| 218 |
#********************************* ACCESS By MONTH ****************************** |
|
| 219 |
if cond_app["is_access_access_by_months"]: |
|
| 220 |
if dico_cste["patternAccessToSeek"] == infos[7]\ |
|
| 221 |
and gi.country_code_by_addr(infos[2]) != "CN": |
|
| 222 |
if (my_time.month, my_time.year) not in dico_access_by_month.keys(): |
|
| 223 |
dico_access_by_month[my_time.month, my_time.year] = 1 |
|
| 224 |
else: |
|
| 225 |
dico_access_by_month[my_time.month, my_time.year] += 1 |
|
| 226 |
#********************************* ACCESS VO By IP ******************************* |
|
| 224 | 227 |
''' |
| 225 | 228 |
if dico_cste["patternVoConeSearchToSeek"] == infos[7] and \ |
| 226 | 229 |
dico_cste["patternVoConeSearchToSeek"] != "": |
| ... | ... | |
| 256 | 259 |
else: |
| 257 | 260 |
access_country_chart_svg_png = None |
| 258 | 261 |
if cond_app["is_access_access_by_months"]: |
| 259 |
access_month_chart_svg_png = svg_bar_chart(dico_access_by_month, start_date, end_date, "Acces par mois"+str(date_title_chart)) |
|
| 262 |
access_month_chart_svg_png = svg_bar_chart(\ |
|
| 263 |
dico_access_by_month, start_date, end_date, "Acces par mois"+str(date_title_chart)) |
|
| 260 | 264 |
else: |
| 261 | 265 |
access_month_chart_svg_png = None |
| 262 | 266 |
if cond_app["is_down_stat_by_project"]: |
| ... | ... | |
| 272 | 276 |
else: |
| 273 | 277 |
total_down_per_ip_chart = None |
| 274 | 278 |
if cond_app["is_down_stat_by_months"]: |
| 275 |
total_down_per_month_chart = svg_bar_chart(total_down_by_months, start_date, end_date,"Volume de telechargement par mois"+str(date_title_chart)) |
|
| 279 |
total_down_per_month_chart = svg_bar_chart(\ |
|
| 280 |
total_down_by_months, start_date, end_date,"Volume de telechargement par mois"+str(date_title_chart)) |
|
| 276 | 281 |
else: |
| 277 | 282 |
total_down_per_month_chart = None |
| 278 | 283 |
if cond_app["is_down_stat_by_country"]: |
Also available in: Unified diff
Mise en place de la nouvelle API Pygal avec le nouveau bar_chart