commit 02482b9acced1720e615b127902998f216adafcd Author: Alessandro_N Date: Mon Apr 27 15:35:42 2015 +0200 Script to monitor datasets status/mapping diff --git a/common/monitoring/README b/common/monitoring/README new file mode 100644 index 0000000..3bf7d17 --- /dev/null +++ b/common/monitoring/README @@ -0,0 +1,34 @@ +# datasets_Monitoring README +# +# Alessandro Nastasi, April 2015 +# + +This is a script aimed at monitoring the status of the Sitools2 datasets +and the mapping of their fields. + +Put the script in the Sitools2 folder data/datasets(/map, for the latest versions) +where the datasets information are stored as xml files. +Then, read and record the current status/mapping, with: + +$ datasets_Monitoring --record + +The datasets properties are locally stored in files named as: + + .lastStatus.xml + +The '--record' option should be executed manually by the administrator every time +one or more datasets are modified (it would be good to add a reminder to the +Sitools2 pop-up message). + +To check the datasets mapping/status, launch the script with the '--check' option: + +$ datasets_Monitoring --check + +This performs a consistency check between the current (int@*.xml or map/string@*.xml) +and the last recorded status (.lastStatus.xml). +If any inconsistency is found, an alert e-mail is sent. + +The '--check' option should be run daily and automatically by the system. + +@author: Alessandro NASTASI for IAS 27-04-2015 + diff --git a/common/monitoring/datasets_Monitoring.py b/common/monitoring/datasets_Monitoring.py new file mode 100755 index 0000000..32854e7 --- /dev/null +++ b/common/monitoring/datasets_Monitoring.py @@ -0,0 +1,317 @@ +#! /usr/bin/python + +# ****************************************************************************** +# Copyright 2015 IAS - IDOC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ****************************************************************************** + +''' +This is a script aimed at monitoring the status of the Sitools2 datasets +and the mapping of their fields. + +Put the script in the Sitools2 folder data/datasets(/map, for the latest versions) +where the datasets information are stored as xml files. +Then, read and record the current status/mapping, with: + +$ datasets_Monitoring --record + +The datasets properties are locally stored in files named as: + + .lastStatus.xml + +The '--record' option should be executed manually by the administrator every time +one or more datasets are modified (it would be good to add a reminder to the +Sitools2 pop-up message). + +To check the datasets mapping/status, launch the script with the '--check' option: + +$ datasets_Monitoring --check + +This performs a consistency check between the current (int@*.xml or map/string@*.xml) +and the last recorded status (.lastStatus.xml). +If any inconsistency is found, an alert e-mail is sent. + +The '--check' option should be run daily and automatically by the system. + +@author: Alessandro NASTASI for IAS -IDOC +@date: 27/04/2015 +''' + +__author__ = "Alessandro Nastasi" +__credits__ = ["Alessandro Nastasi", "Herve' Ballans"] +__license__ = "GPL" +__version__ = "1.0" +__date__ = "27/04/2015" + +import sys,os, time +from datetime import date +import xml.etree.ElementTree as ET +import smtplib +from email.mime.text import MIMEText + +sitools2_xml_filenames = "string@*.xml" ## Files where Sitools stores datasets information +file_path='/usr/local/Sitools2_SZ_Cluster_DB/data/datasets/map/' ## Path where the (string@*.xml) sitools files are stored + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + +_ERROR_CODE = { + 1:'Status inconsistency found:', + 2:'Mapping inconsistency found:', + 3:"The *.lastStatus.xml files were probably not updated: re-run the script with '--record' option." + } + +def create_id_alias_dict(xml_root): + ## Column dictionary " column_id : column_alias ". No mapping information here. + id_alias_dict = {} + for col in xml_root.findall('column'): + column_id = col.find('id').text + column_alias = col.find('columnAlias').text + id_alias_dict[column_id] = column_alias + + return id_alias_dict + + +def send_alert_mail(body): + SMTP_SERVER = 'smtp.ias.u-psud.fr' + SMTP_PORT = 25 + + sender = 'sitools2.notifier@ias.u-psud.fr' + #password = '' + recipient = 'sitools2@ias.u-psud.fr' + subject = '[Sitools2 - SZDB] Datasets status ALERT' + + headers = ["From: "+sender, + "Subject: " + subject, + "To: " + recipient, + "MIME-Version: 1.0", + "Content-Type: text/html"] + headers = "\r\n".join(headers) + + session = smtplib.SMTP(SMTP_SERVER, SMTP_PORT) + + #session.ehlo() + #session.starttls() + #session.ehlo + #session.login(sender, password) + + body = MIMEText(body, 'html') + session.sendmail(sender, recipient, headers + "\r\n\r\n" + body.as_string()) + session.quit() + +def record_status(): + print "\n> Recording current datasets properties ...\n" + + command = "ls "+file_path+sitools2_xml_filenames + intXml_list = os.popen(command).readlines() + + for item in intXml_list: + item=item.strip() + + tree = ET.parse(item) + root = tree.getroot() + + dataset_name = root.find('name').text + fileDataset = file_path+dataset_name+'.lastStatus.xml' + fileOut = open(fileDataset, 'w') + + ##Header in recorded file + today = date.today().strftime("%A %d. %B %Y") + towrite = "\n" + fileOut.write(towrite) + + ##Record the last status of the Dataset + towrite = "\n" + fileOut.write(towrite) + towrite = "\n" + fileOut.write(towrite) + status = root.find('status').text + fileOut.write(" "+status+"") + + ## Column dictionary " column_id : column_alias ". No mapping information here. + column = create_id_alias_dict(root) + + ## Num of mapped concepts + mapped_Concepts_Id = [] + mapped_Column_Id = [] + for elem in root.findall('conceptId'): mapped_Concepts_Id.append(elem.text) + for elem in root.findall('columnId'): mapped_Column_Id.append(elem.text) + + num_mapped_Concepts = len(mapped_Concepts_Id) + + ## Record the last mapping + ## Num of last mapped concepts + towrite = "\n\n \n" + fileOut.write(towrite) + towrite = " "+str(num_mapped_Concepts)+"\n" + fileOut.write(towrite) + + for i,item in enumerate(mapped_Column_Id): + towrite=" "+str(item)+"\n" + fileOut.write(towrite) + towrite=" "+str(column[item])+"\n" + fileOut.write(towrite) + towrite=" "+str(mapped_Concepts_Id[i])+"\n" + fileOut.write(towrite) + + fileOut.write(" \n") + fileOut.close() + print " - Current status and mapping of %s written in %s\n" % (dataset_name, fileDataset) + +def check_status(): + + today = date.today().strftime("%A %d. %B %Y") + now = time.strftime("%H:%M:%S") + check_datime = today +' at '+ now + print "\n#\n#Last check done on", check_datime,"\n#" + print "\n> Checking datasets properties ..." + command = "ls "+file_path+sitools2_xml_filenames + intXml_list = os.popen(command).readlines() + error_status, error_mapping, warning = False, False, False + email_body = "

Outcome of the datasets check done on %s" % check_datime + for item in intXml_list: + item=item.strip() + email_body+= "\n" + tree = ET.parse(item) + currentRoot = tree.getroot() + + dataset_name = currentRoot.find('name').text + filename_dataset = file_path+dataset_name+'.lastStatus.xml' + print "\n - ",dataset_name + + tree = ET.parse(filename_dataset) + lastRoot = tree.getroot() + + #Start the check ... + + ##************ Status ************## + + lastStatus = lastRoot.find('lastStatus').text + currentStatus = currentRoot.find('status').text + output_check = True + showStatus = "" + output_message = "" + + if lastStatus != currentStatus: + if currentStatus == 'INACTIVE': + output_check = bcolors.FAIL+"[FAIL]"+bcolors.ENDC + output_message = "\n "+bcolors.FAIL+_ERROR_CODE[1]+bcolors.ENDC + output_message += "\n Current status: "+bcolors.FAIL+currentStatus+bcolors.ENDC+" - last status: "+bcolors.FAIL+lastStatus+bcolors.ENDC + email_body+="

"+dataset_name+"

*** "+_ERROR_CODE[1]+" ***

Current status: "+currentStatus+" , last status: "+lastStatus+"." + else: + warning = True + output_check = bcolors.WARNING+"[FAIL]"+bcolors.ENDC + output_message = "\n "+bcolors.WARNING+_ERROR_CODE[1]+bcolors.ENDC + output_message += "\n Current status: "+bcolors.WARNING+currentStatus+bcolors.ENDC+" - last status: "+bcolors.WARNING+lastStatus+bcolors.ENDC + email_body+="

"+dataset_name+"

"+_ERROR_CODE[1]+"

Current status: "+currentStatus+" , last status: "+lastStatus+"." + + error_status = True + showStatus="" + + else: + output_check = bcolors.OKGREEN+"[OK]"+bcolors.ENDC + showStatus=" - "+currentStatus + + print ' {0:20s} {1:7s}'.format('Status check ...', output_check)+showStatus+output_message + + + ##************ Mapping ************## + + ## Last mapped concepts + #num_last_mapped_Concepts = int(lastRoot.find('totNum').text) + last_current_mapped_Column_Id = [] + for elem in lastRoot.findall('columnId'): last_current_mapped_Column_Id.append(elem.text) + + ## Current mapped concepts + current_mapped_Column_Id = [] + for elem in currentRoot.findall('columnId'): current_mapped_Column_Id.append(elem.text) + + output_check = "" + + ## Find the columns that lost the mapping + # Create dictionary from current mapping - this does not change from last/current files + column = create_id_alias_dict(currentRoot) + + # Columns mapped in the last file, but not in the current one --> Mapping lost! + missing_current_columnAlias = [column[columnId] for columnId in last_current_mapped_Column_Id if columnId not in current_mapped_Column_Id] + # Columns mapped in the current file, but not in the last one --> User's error: *.lastStatus.xml probably not updated + missing_last_columnAlias = [column[columnId] for columnId in current_mapped_Column_Id if columnId not in last_current_mapped_Column_Id] + + output_message = "" + + if len(missing_current_columnAlias) == 0 and len(missing_last_columnAlias) == 0: + output_check = bcolors.OKGREEN+"[OK]"+bcolors.ENDC + elif len(missing_last_columnAlias) > 0: + warning = True + output_check = bcolors.WARNING+"[FAIL]"+bcolors.ENDC + output_message = "\n "+bcolors.WARNING+_ERROR_CODE[2]+bcolors.ENDC + output_message += "\n Some columns are mapped in the new version, but not in the last one: "+bcolors.WARNING+str(missing_last_columnAlias)+bcolors.ENDC + email_body += "

"+dataset_name+"

"+_ERROR_CODE[2]+"

Some columns are mapped in the new version, but not in the last one: "+str(missing_last_columnAlias)+"

" + elif len(missing_current_columnAlias) > 0 and len(missing_last_columnAlias) == 0: + output_check = bcolors.FAIL+"[FAIL]"+bcolors.ENDC + output_message = "\n "+bcolors.FAIL+_ERROR_CODE[2]+bcolors.ENDC + output_message += "\n The following column(s) is(are) not mapped anymore: "+bcolors.FAIL+str(missing_current_columnAlias)+bcolors.ENDC + email_body += "

"+dataset_name+"

*** "+_ERROR_CODE[2]+" ***

The following column(s) is(are) not mapped anymore: "+str(missing_current_columnAlias)+"

" + + error_mapping = True + + print ' {0:20s} {1:7s}'.format('Mapping check ...', output_check)+output_message + + print "\n__________________________________________________\n\n> Outcome of the check process:" + email_body += "

__________________________________________________

The check process has produced the following message:" + + ## Send an e-mail alert if any error/inconsistency is found + if error_status or error_mapping or warning: + if warning: + print "\n "+bcolors.WARNING+_ERROR_CODE[3]+bcolors.ENDC + email_body += "

"+_ERROR_CODE[3]+'

' + else: + print bcolors.FAIL+'\n Unexpected errors have been found! Please check the datasets properties in Sitools2.'+bcolors.ENDC + email_body += "

Unexpected errors have been found! Please check the datasets properties in Sitools2.


" + + send_alert_mail(email_body) + print "\n\t>> !! ALERT E-MAIL SENT !! <<\n" + + #The alert mail text is stored in an *log.html file for keeping track of the found errors + errors_log_name = file_path+'monitoring_errors.log.html' + errors_log_file = open(errors_log_name, 'a') + errors_log_file.write(email_body) + errors_log_file.close() + + else: + print bcolors.OKGREEN+'\n No errors/inconsistencies found.\n'+bcolors.ENDC + + + +if (len(sys.argv) > 1): + option = sys.argv[1] + if option=='--check': check_status() + elif option=='--record': + overwrite = raw_input(bcolors.WARNING+"\n This option will overwrite the current recorded settings. Do you really want to proceed?: "+ bcolors.ENDC) + if overwrite in ["Yes","yes","Y","y","oui", "OUI", "Oui"]: record_status() + elif overwrite in ["No","no","N", "n"]: + print "Aborted.\n" + exit(0) + else: + print bcolors.WARNING + "\n> Sintax:\t$ python dataset_Monitoring.py [OPTION]\n" + bcolors.ENDC + print "Options:\n\n --check\n\tA status consistency check is performed between the current and last recorded datasets properties." + print "\n --record\n\tThe current datasets properties (status and mapping) are recorded. NB. This procedure will overwrite the previously recorded entries.\n" + exit(0)