This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit 663029937517dc24d2ca44eb3f2bdc466ff38e71 Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Wed Feb 18 14:50:43 2015 +0100 modules can now be dynamically invoked via module_loader --- app/app.py | 21 +- app/module_loader.py | 31 +++ .../__init__.py | 1 + .../nagios.py} | 0 .../snmp.py} | 0 app/modules/connection_modules/ssh.py | 50 ++++ app/modules/detection_modules/drive_detection.py | 34 --- app/modules/detection_modules/nmap_detection.py | 288 ++++++++++----------- app/modules/detection_modules/ssh_detection.py | 103 -------- app/modules/detection_modules/unix/__init__.py | 2 + .../detection_modules/unix/drive_detection.py | 39 +++ .../detection_modules/unix/kernel_detection.py | 14 + app/modules/detection_modules/unix/os_detection.py | 23 ++ .../unix}/__init__.py | 0 app/modules/storage.py | 156 ----------- .../__init__.py | 1 + app/modules/storage_modules/shelve_db.py | 131 ++++++++++ static/js/controllers/detectCtrl.js | 8 +- static/js/controllers/table_ctrl.js | 181 +++++-------- views/dashboard.html | 3 +- views/scan.html | 15 +- 21 files changed, 523 insertions(+), 578 deletions(-) diff --git a/app/app.py b/app/app.py index 29196b2..abba633 100755 --- a/app/app.py +++ b/app/app.py @@ -3,11 +3,11 @@ from __future__ import unicode_literals __author__ = 'aguilbaud' from bottle import * -from modules.detection_modules import nmap_detection from bottle_websocket import GeventWebSocketServer from bottle_websocket import websocket import json import threading +from module_loader import * # Pour lancer la detection nmap avec un nouveau thread @@ -18,12 +18,9 @@ class ThreadDetect(threading.Thread): self.ws = ws def run(self): - req = {} - nmap_detection.check_ip_range(self.ip_range, self.ws) - #print "scanned ip :" - #print get_scanned_ip() - req["20"] = nmap_detection.get_scanned_ip() - self.ws.send(json.dumps(req)) + db = load_db() + scanned_ip = run_nmap_detection(self.ip_range, db, self.ws) + self.ws.send(json.dumps({"20" : scanned_ip})) @route('/') @@ -74,7 +71,6 @@ def angular(): # Lancement de la detection apres reception d'une plage d'ip def start_first_detection(ip_range, ws): - req = {} # Verification que la plage d'ip est bien formatee if re.search('^\d{1,3}(-\d{1,3})?[.]\d{1,3}(-\d{1,3})?[.]\d{1,3}(-\d{1,3})?[.]\d{1,3}(-\d{1,3})?$', ip_range): # Si oui on lance la detection avec le module nmap_detection @@ -82,8 +78,7 @@ def start_first_detection(ip_range, ws): t.start() else: # Si non, on envoie un message d'erreur - req["40"] = "La plage d'IP est mal formatée" - ws.send(json.dumps(req)) + ws.send(json.dumps({"40": "Ip range incorrectly formatted"})) @error(404) def error404(error): @@ -112,8 +107,12 @@ def receive(ws): for code in msg: if code == "10": start_first_detection(msg["10"], ws) + elif code == "14": + db = load_db() + ws.send(json.dumps({"22": db.get_hosts()})) + del db else: - return + break diff --git a/app/module_loader.py b/app/module_loader.py new file mode 100644 index 0000000..ccde508 --- /dev/null +++ b/app/module_loader.py @@ -0,0 +1,31 @@ +__author__ = 'aguilbaud' +import modules.detection_modules +import modules.detection_modules.unix +import modules.connection_modules +import modules.storage_modules + + +def load_db(): + db_name = "shelve_db" + db = __import__("modules.storage_modules." + db_name, fromlist=modules.storage_modules) + db_instance = getattr(db, db_name)() + return db_instance + + +def run_nmap_detection(ip_range, db, ws): + nmap_mod = __import__("modules.detection_modules.nmap_detection", fromlist=modules.detection_modules) + nmap_mod_instance = getattr(nmap_mod, "nmap_detection")(db, ws) + return nmap_mod_instance.check_ip_range(ip_range) + + +def load_conn(conn_name, addr_host, key_location): # /home/aguilbaud/.ssh/id_rsa + conn = __import__("modules.connection_modules." + conn_name, fromlist=modules.connection_modules) + conn_instance = getattr(conn, conn_name)(addr_host, key_location) + return conn_instance + + +def run_all_detection_modules(os, conn, db): + for mod_name in "modules.detection_modules." + os + ".__all__": + mod = __import__ ("modules.detection_modules." + os + "." + mod_name, fromlist=modules.detection_modules.unix.__all__) # on charge le module + mod_instance = getattr(mod, mod_name)(conn, db) # on appelle le constructeur + mod_instance.run_detection() \ No newline at end of file diff --git a/app/modules/storage_module/__init__.py b/app/modules/connection_modules/__init__.py similarity index 59% copy from app/modules/storage_module/__init__.py copy to app/modules/connection_modules/__init__.py index fcb43f2..ef5647c 100644 --- a/app/modules/storage_module/__init__.py +++ b/app/modules/connection_modules/__init__.py @@ -1 +1,2 @@ __author__ = 'aguilbaud' +__all__ = ['ssh'] \ No newline at end of file diff --git a/app/modules/detection_modules/nagios_detection.py b/app/modules/connection_modules/nagios.py similarity index 100% rename from app/modules/detection_modules/nagios_detection.py rename to app/modules/connection_modules/nagios.py diff --git a/app/modules/detection_modules/snmp_detection.py b/app/modules/connection_modules/snmp.py similarity index 100% rename from app/modules/detection_modules/snmp_detection.py rename to app/modules/connection_modules/snmp.py diff --git a/app/modules/connection_modules/ssh.py b/app/modules/connection_modules/ssh.py new file mode 100644 index 0000000..b4473b4 --- /dev/null +++ b/app/modules/connection_modules/ssh.py @@ -0,0 +1,50 @@ +import paramiko + + +class ssh: + + def __init__(self, addr_host, key_location): + key = paramiko.RSAKey.from_private_key_file(key_location) # "/home/aguilbaud/.ssh/id_rsa" + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.ssh.connect(addr_host, username='aguilbaud', pkey=key) + self.addr_host = addr_host + + def get_addr_host(self): + return self.addr_host + + def exec_command(self, cmd): + stdin, stdout, stderr = self.ssh.exec_command(cmd) + res = stdout.read() + return res + + def disconnect(self): + self.ssh.close() + + + ''' + # Informations materielles globales + def detect_hardware(ssh): + cmd = "lshw -json" + stdin, stdout, stderr = ssh.exec_command(cmd) + res = "" + for line in stdout.read().splitlines(): + res += line + res_json = json.loads(res) + # TODO Traitement du resultat pour garder l'essentiel... + return res_json + + + + + # dependant de la langue du systeme ? + def detect_non_updated_packages(ssh): + cmd = "apt-get upgrade -s" + stdin, stdout, stderr = ssh.exec_command(cmd) + res = stdout.read() + tab_res = res.split(':') + if len(tab_res) == 2: + return json.dumps({'non_updated_packages': False}) + else: + return json.dumps({'non_updated_packages': True}) + ''' \ No newline at end of file diff --git a/app/modules/detection_modules/drive_detection.py b/app/modules/detection_modules/drive_detection.py deleted file mode 100644 index fb374b3..0000000 --- a/app/modules/detection_modules/drive_detection.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf8 -*- -from __future__ import unicode_literals -__author__ = 'aguilbaud' - -''' -Retourne les informations des partitions systeme sous la forme : -{"sr0": {"mountpoint": "none", "type": "rom", "name": "sr0", "size": "1024M"} -''' - -# Informations sur les partitions -def detect_drives(conn, os): - cmd = "lsblk -r --output=NAME,SIZE,TYPE,MOUNTPOINT" - stdin, stdout, stderr = conn.exec_command(cmd) - dict_total = {} - i = 1 - ignore = True - for line in stdout.read().splitlines(): - # On ignore la premiere ligne qui ne contient pas de valeurs - if ignore: - ignore = False - else: - dict_drive = {} - tab_elem = line.split() - dict_drive["name"] = tab_elem[0] - dict_drive["size"] = tab_elem[1] - dict_drive["type"] = tab_elem[2] - if len(tab_elem) > 3: - dict_drive["mountpoint"] = tab_elem[3] - else: - dict_drive["mountpoint"] = "none" - # meilleur nom pour chaque attribut ? - dict_total[dict_drive["name"]] = dict_drive - i = i + 1 - return json.dumps(dict_total) \ No newline at end of file diff --git a/app/modules/detection_modules/nmap_detection.py b/app/modules/detection_modules/nmap_detection.py index d5a95c1..9570695 100644 --- a/app/modules/detection_modules/nmap_detection.py +++ b/app/modules/detection_modules/nmap_detection.py @@ -1,160 +1,140 @@ -# -*- coding: utf8 -*- -from __future__ import unicode_literals __author__ = 'aguilbaud' from xml.dom import minidom import pexpect import json -import shelve -import os.path -from .. import storage - -scanned_ip = {} - - -def get_scanned_ip(): - global scanned_ip - return json.dumps(scanned_ip) - - -# fonction qui permet de decomposer les differentes plages d'ip -# lance la detection nmap pour chacune des ip comprises dans cette plage -# NB : il est possible de lancer nmap directement sur la plage, mais il est alors difficile de determiner l'état -# du scan. De plus, si la plage d'ip est grande, l'exécution sera bien plus longue et tous les résultats -# seront obtenus d'un coup à la fin. -def check_ip_range(ip_range, ws): - # separation des 4 octets - range_byte_1 = ip_range.split('.')[0] - range_byte_2 = ip_range.split('.')[1] - range_byte_3 = ip_range.split('.')[2] - range_byte_4 = ip_range.split('.')[3] - - # separation des plages eventuelles - split_byte_1 = range_byte_1.split('-') - split_byte_2 = range_byte_2.split('-') - split_byte_3 = range_byte_3.split('-') - split_byte_4 = range_byte_4.split('-') - - # si aucune plage n'est indiquee, on cree une plage de meme valeur - if len(split_byte_1) == 1: - split_byte_1.append(split_byte_1[0]) - # verification que les nombres sont ordonnes correctement - # et que les valeurs entrees sont inferieures a 255 - split_byte_1 = check_order_and_under_255(split_byte_1) - - # idem pour le deuxieme octet - if len(split_byte_2) == 1: - split_byte_2.append(split_byte_2[0]) - split_byte_2 = check_order_and_under_255(split_byte_2) - - # idem pour le troisieme octet - if len(split_byte_3) == 1: - split_byte_3.append(split_byte_3[0]) - split_byte_3 = check_order_and_under_255(split_byte_3) - - # idem pour le quatrieme octet - if len(split_byte_4) == 1: - split_byte_4.append(split_byte_4[0]) - split_byte_4 = check_order_and_under_255(split_byte_4) - - # il est possible d'ajouter ici une condition pour verifier que l'utilisateur - # n'ait pas entre une plage d'IP trop grande - - # pour toutes les plages dans l'ordre croissant, en partant du dernier octet - for byte_1 in range(int(split_byte_1[0]), int(split_byte_1[1]) + 1): - for byte_2 in range(int(split_byte_2[0]), int(split_byte_2[1]) + 1): - for byte_3 in range(int(split_byte_3[0]), int(split_byte_3[1]) + 1): - for byte_4 in range(int(split_byte_4[0]), int(split_byte_4[1]) + 1): - launch_detection(byte_1, byte_2, byte_3, byte_4, ws) - - -# vérifie que la plage de données entrée est dans l'ordre croissant -# et que ses valeurs sont inferieures à 255 -# si ce n'est pas le cas, retourne le tableau trié et/ou avec les valeurs capées à 255 -def check_order_and_under_255(tab_val): - if int(tab_val[0]) > 255 : - tab_val[0] = '255' - if int(tab_val[1]) > 255 : - tab_val[1] = '255' - if int(tab_val[0]) > int(tab_val[1]): - tmp = tab_val[1] - tab_val[1] = tab_val[0] - tab_val[0] = tmp - return tab_val - - -# lance la detection a l'aide de nmap sur l'ip representee par les 4 octets passes en parametres -def launch_detection(byte_1, byte_2, byte_3, byte_4, ws): - req = {} - ip = str(byte_1) + '.' + str(byte_2) + '.' + str(byte_3) + '.' + str(byte_4) - req["30"] = "Scan de l'IP " + ip + " en cours..." - ws.send(json.dumps(req)) - child = pexpect.spawn('nmap', ['-A', ip, '-oX', 'res.xml']) - res = '' - # ici : possibilite de verifier l'avancement du scan, si option verbose (-v3) activee dans la commande nmap - try: - while child.isalive(): - child.expect('Completed', timeout=None) - res += child.before + '<br/>' - except pexpect.EOF: - res += ' A FINI' + + +class nmap_detection: + def __init__(self, db, ws): + self.db = db + self.ws = ws + self.scanned_ip = [] + + # function for splitting the different ranges of the IP adress + # launch the nmap detection of each ip under this range + # NB : it is possible to launch nmap directly on the range, but it becomes difficult to determine the state + # of the scan. Moreover,if the ip range is very big, the execution will take more time and each result + # will be avaliable only a the end of the scan of the range. + def check_ip_range(self, ip_range): + # separation of the 4 bytes + range_byte_1 = ip_range.split('.')[0] + range_byte_2 = ip_range.split('.')[1] + range_byte_3 = ip_range.split('.')[2] + range_byte_4 = ip_range.split('.')[3] + + # separation of eventual ranges + split_byte_1 = range_byte_1.split('-') + split_byte_2 = range_byte_2.split('-') + split_byte_3 = range_byte_3.split('-') + split_byte_4 = range_byte_4.split('-') + + # if no range is indicated, we create one of same value + if len(split_byte_1) == 1: + split_byte_1.append(split_byte_1[0]) + # checking that numbers are ordored correctly + # and that there values are under 255 + split_byte_1 = self.check_order_and_under_255(split_byte_1) + + # same for the second byte + if len(split_byte_2) == 1: + split_byte_2.append(split_byte_2[0]) + split_byte_2 = self.check_order_and_under_255(split_byte_2) + + # same for the third byte + if len(split_byte_3) == 1: + split_byte_3.append(split_byte_3[0]) + split_byte_3 = self.check_order_and_under_255(split_byte_3) + + # same for the fourth byte + if len(split_byte_4) == 1: + split_byte_4.append(split_byte_4[0]) + split_byte_4 = self.check_order_and_under_255(split_byte_4) + + # it's possible here to check the size of the range + + # for each range in increasing order, beginning by the last byte + for byte_1 in range(int(split_byte_1[0]), int(split_byte_1[1]) + 1): + for byte_2 in range(int(split_byte_2[0]), int(split_byte_2[1]) + 1): + for byte_3 in range(int(split_byte_3[0]), int(split_byte_3[1]) + 1): + for byte_4 in range(int(split_byte_4[0]), int(split_byte_4[1]) + 1): + self.launch_detection(byte_1, byte_2, byte_3, byte_4) + # once finished, returns the list of scanned ip + return json.dumps(self.scanned_ip) + + # check that the numbers are ordered increasing + # and their values are under 255 + # if it is not the case, returns the list in increasing order, and/or with values capped at 255 + def check_order_and_under_255(self, tab_val): + if int(tab_val[0]) > 255: + tab_val[0] = '255' + if int(tab_val[1]) > 255: + tab_val[1] = '255' + if int(tab_val[0]) > int(tab_val[1]): + tmp = tab_val[1] + tab_val[1] = tab_val[0] + tab_val[0] = tmp + return tab_val + + # launch the nmap detection for the IP adress represented by the 4 bytes in parameters + def launch_detection(self, byte_1, byte_2, byte_3, byte_4): + ip = str(byte_1) + '.' + str(byte_2) + '.' + str(byte_3) + '.' + str(byte_4) + self.ws.send(json.dumps({"30": "Scanning ip : " + ip})) + child = pexpect.spawn('nmap', ['-A', ip, '-oX', 'res.xml']) + # here : possible to check the advancement of the scan, by putting verbose "-v3" option on command try: - parse_res(ip) - except: - del req["30"] - req["40"] = "Database error" - except pexpect.TIMEOUT: - del req["30"] - req["40"] = "Timeout on nmap execution" - ws.send(json.dumps(req)) - return res - - -# parse le resultat xml de nmap pour ne conserver que les valeurs interssantes -# envoie directement le resultat sur le service ElasticSearch -def parse_res(ip): - global scanned_ip - # Ouverture du fichier xml avec le parseur minidom - root = minidom.parse("res.xml") - collection = root.documentElement - - # Recuperer tous les <host> de la collection - hosts = collection.getElementsByTagName("host") - - # Recuperation des noeuds de chaque <host> et affichage de leur attributss - # JSON = liste de dictionnaires - for host in hosts: - status = host.getElementsByTagName('status')[0] - address = host.getElementsByTagName('address')[0] - - dict_host = {} - dict_host['addr'] = address.getAttribute('addr') - dict_host['date'] = host.getAttribute('endtime') - dict_host['state'] = status.getAttribute('state') - dict_host['os'] = 'unknown' # par defaut - - hostnames_elem = host.getElementsByTagName('hostnames')[0] - hostnames = hostnames_elem.getElementsByTagName('hostname') - for hostname in hostnames: - dict_host['hostname'] = hostname.getAttribute("name") - - ports_elem = host.getElementsByTagName('ports')[0] - ports = ports_elem.getElementsByTagName('port') - list_dict_port = [] - for port in ports: - dict_port = {} - state = port.getElementsByTagName('state')[0] - service = port.getElementsByTagName('service')[0] - if service.hasAttribute("ostype"): - dict_host['os'] = service.getAttribute("ostype") - if state.getAttribute('state') == 'open': - dict_port['portid'] = port.getAttribute('portid') - dict_port['portname'] = service.getAttribute('name') - # Ajouter d'autres infos ? - list_dict_port.append(dict_port) - dict_host['openports'] = list_dict_port - # sauvegarde de l'host dans la base elasticsearch avec pour ID son IP - storage.add_host(dict_host['addr'], json.dumps(dict_host)) - pexpect.run("rm -f res.xml") - #pexpect.run('curl -XPUT \'localhost:9200/saved_hosts' - scanned_ip[ip] = "localhost:9200/host/external/" + ip \ No newline at end of file + while child.isalive(): + child.expect('Completed', timeout=None) + except pexpect.EOF: + try: + self.parse_res(ip) + except: + self.ws.send(json.dumps({"40": "Database error"})) + except pexpect.TIMEOUT: + self.ws.send(json.dumps({"40": "Timeout on nmap execution"})) + + # parse the xml result to keep only interesting values + # save directly it on the database + def parse_res(self, ip): + # opening the xml file with minidom parser + root = minidom.parse("res.xml") + collection = root.documentElement + + # get every <host> of the collection + hosts = collection.getElementsByTagName("host") + + # Get the nodes of each <host> and recuperaton of their attributes + # JSON = dictionary list + for host in hosts: + status = host.getElementsByTagName('status')[0] + address = host.getElementsByTagName('address')[0] + + dict_host = {} + dict_host['addr'] = address.getAttribute('addr') + dict_host['date'] = host.getAttribute('endtime') + dict_host['state'] = status.getAttribute('state') + dict_host['os'] = 'unknown' # par defaut + + hostnames_elem = host.getElementsByTagName('hostnames')[0] + hostnames = hostnames_elem.getElementsByTagName('hostname') + for hostname in hostnames: + dict_host['hostname'] = hostname.getAttribute("name") + + ports_elem = host.getElementsByTagName('ports')[0] + ports = ports_elem.getElementsByTagName('port') + list_dict_port = [] + for port in ports: + dict_port = {} + state = port.getElementsByTagName('state')[0] + service = port.getElementsByTagName('service')[0] + if service.hasAttribute("ostype"): + dict_host['os'] = service.getAttribute("ostype") + if state.getAttribute('state') == 'open': + dict_port['portid'] = port.getAttribute('portid') + dict_port['portname'] = service.getAttribute('name') + list_dict_port.append(dict_port) + dict_host['openports'] = list_dict_port + # the host have its IP for ID on the db + self.db.add_host(dict_host['addr'], json.dumps(dict_host)) + pexpect.run("rm -f res.xml") + self.scanned_ip.append(ip) \ No newline at end of file diff --git a/app/modules/detection_modules/ssh_detection.py b/app/modules/detection_modules/ssh_detection.py deleted file mode 100644 index c2d1c52..0000000 --- a/app/modules/detection_modules/ssh_detection.py +++ /dev/null @@ -1,103 +0,0 @@ -import paramiko -import json - - -# il faudrait p-e separer le traitement du resultat des commandes avec le lancement en lui mm -# si independant des protocoles utilises - -def connect(): - # TODO rendre la connection dynamique - key = paramiko.RSAKey.from_private_key_file("/home/aguilbaud/.ssh/id_rsa") - - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect('127.0.0.1', username='aguilbaud', pkey=key) - - run_detection(ssh) - - disconnect(ssh) - - -def disconnect(ssh): - ssh.close() - - -def run_detection(ssh): - print detect_hardware(ssh) - print detect_drives(ssh) - print detect_os_version(ssh) - print detect_non_updated_packages(ssh) - - -# Informations materielles globales -def detect_hardware(ssh): - cmd = "lshw -json" - stdin, stdout, stderr = ssh.exec_command(cmd) - res = "" - for line in stdout.read().splitlines(): - res += line - res_json = json.loads(res) - # TODO Traitement du resultat pour garder l'essentiel... - return res_json - - -# Informations sur les partitions -def detect_drives(ssh): - cmd = "lsblk -r --output=NAME,SIZE,TYPE,MOUNTPOINT" - stdin, stdout, stderr = ssh.exec_command(cmd) - dict_total = {} - i = 1 - ignore = True - for line in stdout.read().splitlines(): - # On ignore la premiere ligne qui ne contient pas de valeurs - if ignore: - ignore = False - else: - dict_drive = {} - tab_elem = line.split() - dict_drive["name"] = tab_elem[0] - dict_drive["size"] = tab_elem[1] - dict_drive["type"] = tab_elem[2] - if len(tab_elem) > 3: - dict_drive["mountpoint"] = tab_elem[3] - else: - dict_drive["mountpoint"] = "none" - # meilleur nom pour chaque attribut ? - dict_total[dict_drive["name"]] = dict_drive - i = i + 1 - return json.dumps(dict_total) - - -def detect_os_version(ssh): - # kernel - cmd = "cat /proc/version" - stdin, stdout, stderr = ssh.exec_command(cmd) - res = stdout.read() - dict_total = {} - dict_total["kernel"] = res.split('#')[0] - - # os - cmd = "cat /etc/os-release" - stdin, stdout, stderr = ssh.exec_command(cmd) - for line in stdout.read().splitlines(): - tab_elem = line.split("=") - # pour retirer les "" sur tous les champs qui en possedent - tab_right = tab_elem[1].split('"') - if len(tab_right) == 1: - dict_total[str.lower(tab_elem[0])] = tab_right[0] - else: - dict_total[str.lower(tab_elem[0])] = tab_right[1] - # encore une fois, on recupere tout le contenu de la commande, p-e qu'il est possible d'enlever le superflu - return json.dumps(dict_total, indent=4, separators=(',', ': ')) - - -# dependant de la langue du systeme ? -def detect_non_updated_packages(ssh): - cmd = "apt-get upgrade -s" - stdin, stdout, stderr = ssh.exec_command(cmd) - res = stdout.read() - tab_res = res.split(':') - if len(tab_res) == 2: - return json.dumps({'non_updated_packages': False}) - else: - return json.dumps({'non_updated_packages': True}) \ No newline at end of file diff --git a/app/modules/detection_modules/unix/__init__.py b/app/modules/detection_modules/unix/__init__.py new file mode 100644 index 0000000..cb82579 --- /dev/null +++ b/app/modules/detection_modules/unix/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'aguilbaud' +__all__ = ['drive_detection', 'kernel_detection', 'os_detection'] \ No newline at end of file diff --git a/app/modules/detection_modules/unix/drive_detection.py b/app/modules/detection_modules/unix/drive_detection.py new file mode 100644 index 0000000..18d548e --- /dev/null +++ b/app/modules/detection_modules/unix/drive_detection.py @@ -0,0 +1,39 @@ +__author__ = 'aguilbaud' +import json + + +class drive_detection: + def __init__(self, conn, db): + self.conn = conn + self.db = db + + ''' + Retourne les informations des partitions systeme sous la forme : + {"sr0": {"mountpoint": "none", "type": "rom", "name": "sr0", "size": "1024M"} + ''' + + # Informations sur les partitions + def run_detection(self): + cmd = "lsblk -r --output=NAME,SIZE,TYPE,MOUNTPOINT" + stdout = self.conn.exec_command(cmd) + dict_total = {} + i = 1 + ignore = True + for line in stdout.splitlines(): + # On ignore la premiere ligne qui ne contient pas de valeurs + if ignore: + ignore = False + else: + dict_drive = {} + tab_elem = line.split() + dict_drive["name"] = tab_elem[0] + dict_drive["size"] = tab_elem[1] + dict_drive["type"] = tab_elem[2] + if len(tab_elem) > 3: + dict_drive["mountpoint"] = tab_elem[3] + else: + dict_drive["mountpoint"] = "none" + # meilleur nom pour chaque attribut ? + dict_total[dict_drive["name"]] = dict_drive + i += 1 + self.db.save_detection(self.conn.get_addr_host(), "drive_detection", json.dumps(dict_total)) \ No newline at end of file diff --git a/app/modules/detection_modules/unix/kernel_detection.py b/app/modules/detection_modules/unix/kernel_detection.py new file mode 100644 index 0000000..037098a --- /dev/null +++ b/app/modules/detection_modules/unix/kernel_detection.py @@ -0,0 +1,14 @@ +__author__ = 'aguilbaud' +import json + +class kernel_detection: + def __init__(self, conn, db): + self.conn = conn + self.db = db + + def run_detection(self): + cmd = "cat /proc/version" + stdout = self.conn.exec_command(cmd) + dict_total = {} + dict_total["kernel"] = stdout.split('#')[0] + self.db.save_detection(self.conn.get_addr_host(), "kernel_detection", json.dumps(dict_total)) \ No newline at end of file diff --git a/app/modules/detection_modules/unix/os_detection.py b/app/modules/detection_modules/unix/os_detection.py new file mode 100644 index 0000000..6126250 --- /dev/null +++ b/app/modules/detection_modules/unix/os_detection.py @@ -0,0 +1,23 @@ +__author__ = 'aguilbaud' +import json + + +class os_detection: + def __init__(self, conn, db): + self.conn = conn + self.db = db + + def run_detection(self): + dict_total = {} + cmd = "cat /etc/os-release" + stdout = self.conn.exec_command(cmd) + for line in stdout.splitlines(): + tab_elem = line.split("=") + # pour retirer les "" sur tous les champs qui en possedent + tab_right = tab_elem[1].split('"') + if len(tab_right) == 1: + dict_total[str.lower(tab_elem[0])] = tab_right[0] + else: + dict_total[str.lower(tab_elem[0])] = tab_right[1] + # encore une fois, on recupere tout le contenu de la commande, p-e qu'il est possible d'enlever le superflu + self.db.save_detection(self.conn.get_addr_host(), "os_detection", json.dumps(dict_total)) \ No newline at end of file diff --git a/app/modules/storage_module/__init__.py b/app/modules/monitoring_modules/unix/__init__.py similarity index 100% copy from app/modules/storage_module/__init__.py copy to app/modules/monitoring_modules/unix/__init__.py diff --git a/app/modules/storage.py b/app/modules/storage.py deleted file mode 100644 index d4d7d06..0000000 --- a/app/modules/storage.py +++ /dev/null @@ -1,156 +0,0 @@ -__author__ = 'aguilbaud' - -import shelve -from datetime import datetime -from math import sqrt -import os.path -import json - - -# Returns an instance of the python database -# If the base doesn't exists, it initialize the first elements -def init_db(): - if not os.path.isfile("mum.db"): - db = shelve.open("mum.db", writeback=True) - try: - db["hosts"] = {} - db["users"] = {} - db["groups"] = {} - db["global_conf"] = {} - finally: - return db - return shelve.open("mum.db", writeback=True) - - -# Closes the database -def close_db(db): - db.close() - - -# Add and save a new host after its first nmap detection -# It also preconfigure with the default configuration, add the host to the group "all" and -# creates empty structures for the monitoring and archive data. -def add_host(addr_host, nmap_res): - addr_host = str(addr_host) # Shelve doesn't support Unicode - db = init_db() - try: - # Add the nmap detection - db["hosts"][addr_host] = {} - db["hosts"][addr_host]["detected"] = {} - db["hosts"][addr_host]["detected"]["nmap"] = nmap_res - # Preconfiguration - db["hosts"][addr_host]["conf"] = {} - db["hosts"][addr_host]["conf"]["monitoring"] = db["global_conf"] - db["hosts"][addr_host]["conf"]["groups"] = {"name": "all"} - db["hosts"][addr_host]["conf"]["subscribers"] = {} # Add current user automatically ? - db["hosts"][addr_host]["conf"]["custom_info"] = "" - db["hosts"][addr_host]["conf"]["interventions"] = {} - # Create structure for monitoring data - db["hosts"][addr_host]["monitoring"] = {} - # Create structure for archiving data - db["hosts"][addr_host]["archive"] = {} - finally: - close_db(db) - - -# Returns the essential data about all hosts under monitoring -# These are used by the front-end -# If no hosts have been added, the function will return an empty list -def get_hosts(): - res = [] - db = init_db() - try: - if db["hosts"] != {}: - for host in db["hosts"]: - detected = json.loads(db["hosts"][host]["detected"]["nmap"]) - info_host = {} - info_host["addr"] = detected["addr"] - info_host["name"] = detected["hostname"] - if "status" in db["hosts"][host]["monitoring"]: - info_host["status"] = db["hosts"][host]["monitoring"]["status"] - else: - info_host["status"] = "" - info_host["group"] = [] - for group in db["hosts"][host]["conf"]["groups"]: - info_host["group"].append({"name": db["hosts"][host]["conf"]["groups"][group]}) - if "date" in db["hosts"][host]["monitoring"]: - info_host["last_check"] = db["hosts"][host]["monitoring"]["date"] - else: - info_host["last_check"] = 0 - res.append(info_host) - finally: - close_db(db) - return json.dumps(res) - - -# Add a new check of a host from a specific module -def add_check(addr_host, name_part, val): - new_val = {"date": datetime.now()} - db = init_db() - try: - if val >= db['hosts']['conf']['monitorig'][name_part]['minor_limit']: - new_val['state'] = 'warning' - elif val >= db['hosts']['conf']['monitorig'][name_part]['major_limit']: - new_val['state'] = 'danger' - else: - new_val['state'] = 'success' - previous_val = db['hosts'][addr_host]["monitoring"][name_part] - db['hosts'][addr_host]['monitoring'][name_part] = new_val - # now performing archiving - if db['hosts'][addr_host]['archive'].has_key(name_part): - db['hosts'][addr_host]['archive'][name_part] = update_stats(db['hosts'][addr_host]['archive'][name_part], val) - finally: - close_db(db) - - -# Updates calulated statistics once a new value is received -def update_stats(stats, val): - stats['nb_check'] += 1 - stats['total'] += val - if stats['min'] > val: - stats['min'] = val - if stats['max'] < val: - stats['max'] = val - # Compute linear regression - stats['lr'] += val * stats['nb_check'] - # Compute variance - stats['delta'] = val - stats['mean'] - stats['mean'] += stats['delta'] / stats['nb_check'] - stats['M2'] += stats['delta'] * (val - stats['mean']) - return stats - - -def get_mean(stats): - res = float('nan') - try: - res = stats['total'] / stats['nb_check'] - except ZeroDivisionError: - print "Division by 0 on get_mean(stats)" - return res - - -def get_standard_derivation(stats): - variance = stats['M2'] / max(1, stats['nb_check'] + 1) - res = sqrt(variance) - return res - - -def get_slope_of_linear_regression(stats): - res = float('nan') - try: - mX = (stats['nb_check'] + 1) / 2 - mY = stats['total'] / stats['nb_check'] - mX2 = (stats['nb_check'] + 1) * (2 * stats['nb_check'] + 1) / 6 - mXY = stats['lr'] / stats['nb_check'] - res = (mXY - mX * mY) / (mX2 - mX * mX) - except ZeroDivisionError: - print "Division by 0 on get_slope_of_linear_regression(stats)" - return res - - -def save_detection(name_part, json_res): - db = shelve.open('mum.db', writeback=True) - try: - return - finally: - db.close() \ No newline at end of file diff --git a/app/modules/storage_module/__init__.py b/app/modules/storage_modules/__init__.py similarity index 52% rename from app/modules/storage_module/__init__.py rename to app/modules/storage_modules/__init__.py index fcb43f2..9ec9db7 100644 --- a/app/modules/storage_module/__init__.py +++ b/app/modules/storage_modules/__init__.py @@ -1 +1,2 @@ __author__ = 'aguilbaud' +__all__ = ['shelve_db'] \ No newline at end of file diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py new file mode 100644 index 0000000..7cc71dd --- /dev/null +++ b/app/modules/storage_modules/shelve_db.py @@ -0,0 +1,131 @@ +__author__ = 'aguilbaud' + +from datetime import datetime +import json +import shelve + + +import os.path + + +class shelve_db: + + def __init__(self): + self.db = None + + def open_db(self): + if not os.path.isfile("mum.db"): # init of the database at the first opening + self.db = shelve.open("mum.db", writeback=True) + try: + self.db["hosts"] = {} + self.db["users"] = {} + self.db["groups"] = {} + self.db["global_conf"] = {} + except: + print "Database initilalization error" + else: + self.db = shelve.open("mum.db", writeback=True) + + # Closes the database + def close_db(self): + self.db.close() + self.db = None + + + # Add and save a new host after its first nmap detection + # It also preconfigure with the default configuration, add the host to the group "all" and + # creates empty structures for the monitoring and archive data. + def add_host(self,addr_host, nmap_res): + self.open_db() + addr_host = str(addr_host) # Shelve doesn't support Unicode + try: + # Add the nmap detection + self.db["hosts"][addr_host] = {} + self.db["hosts"][addr_host]["detected"] = {} + self.db["hosts"][addr_host]["detected"]["nmap"] = nmap_res + # Preconfiguration + self.db["hosts"][addr_host]["conf"] = {} + self.db["hosts"][addr_host]["conf"]["monitoring"] = self.db["global_conf"] + self.db["hosts"][addr_host]["conf"]["groups"] = {"name": "all"} + self.db["hosts"][addr_host]["conf"]["subscribers"] = {} # Add current user automatically ? + self.db["hosts"][addr_host]["conf"]["custom_info"] = "" + self.db["hosts"][addr_host]["conf"]["interventions"] = {} + # Create structure for monitoring data + self.db["hosts"][addr_host]["monitoring"] = {} + # Create structure for archiving data + self.db["hosts"][addr_host]["archive"] = {} + finally: + self.close_db() + + + # Returns the essential data about all hosts under monitoring + # These are used by the front-end + # If no hosts have been added, the function will return an empty list + def get_hosts(self): + self.open_db() + res = [] + try: + if self.db["hosts"] != {}: + for host in self.db["hosts"]: + detected = json.loads(self.db["hosts"][host]["detected"]["nmap"]) + info_host = {} + info_host["addr"] = detected["addr"] + info_host["name"] = detected["hostname"] + if "status" in self.db["hosts"][host]["monitoring"]: + info_host["status"] = self.db["hosts"][host]["monitoring"]["status"] + else: + info_host["status"] = "" + info_host["group"] = [] + for group in self.db["hosts"][host]["conf"]["groups"]: + info_host["group"].append({"name": self.db["hosts"][host]["conf"]["groups"][group]}) + if "date" in self.db["hosts"][host]["monitoring"]: + info_host["last_check"] = self.db["hosts"][host]["monitoring"]["date"] + else: + info_host["last_check"] = 0 + res.append(info_host) + finally: + self.close_db() + return json.dumps(res) + + def save_detection(self, addr_host, name_part, json_res_str): + self.open_db() + try: + self.db["hosts"][addr_host]["detected"][name_part] = json_res_str + finally: + self.close_db() + + # Add a new check of a host from a specific module + def add_check(self, addr_host, name_part, val): + self.open_db() + new_val = {"date": datetime.now()} + try: + if val >= self.db['hosts']['conf']['monitorig'][name_part]['minor_limit']: + new_val['state'] = 'warning' + elif val >= self.db['hosts']['conf']['monitorig'][name_part]['major_limit']: + new_val['state'] = 'danger' + else: + new_val['state'] = 'success' + previous_val = self.db['hosts'][addr_host]["monitoring"][name_part] + self.db['hosts'][addr_host]['monitoring'][name_part] = new_val + # now performing archiving + if self.db['hosts'][addr_host]['archive'].has_key(name_part): + self.db['hosts'][addr_host]['archive'][name_part] = \ + self.update_stats(self.db['hosts'][addr_host]['archive'][name_part], val) + finally: + self.close_db() + + # Updates calulated statistics once a new value is received + def update_stats(self, stats, val): + stats['nb_check'] += 1 + stats['total'] += val + if stats['min'] > val: + stats['min'] = val + if stats['max'] < val: + stats['max'] = val + # Compute linear regression + stats['lr'] += val * stats['nb_check'] + # Compute variance + stats['delta'] = val - stats['mean'] + stats['mean'] += stats['delta'] / stats['nb_check'] + stats['M2'] += stats['delta'] * (val - stats['mean']) + return stats \ No newline at end of file diff --git a/static/js/controllers/detectCtrl.js b/static/js/controllers/detectCtrl.js index 69afe93..deed7f1 100644 --- a/static/js/controllers/detectCtrl.js +++ b/static/js/controllers/detectCtrl.js @@ -1,6 +1,6 @@ var formExample = angular.module('detectModule', ['toastr']); -formExample.controller('DetectController', ['$scope', 'toastr', '$http', '$interval', function($scope, toastr, $http, $interval) { +formExample.controller('DetectController', ['$scope', 'toastr', '$interval', function($scope, toastr, $interval) { $scope.master = {}; $scope.ip_range = "198.116.0.1-10" // la plage d'ip entree dans le champ $scope.state = ""; // l'etat general du scan en cours @@ -18,12 +18,18 @@ formExample.controller('DetectController', ['$scope', 'toastr', '$http', '$inter JSON.parse(evt.data, function (key, value) { switch(parseInt(key)){ case 20: // Success of a module execution + $scope.$apply(function(){ + $scope.state = "Success!"; + }); toastr.success(value, "Success on module execution"); case 21: // Informations concerning one host break; case 22: // List of hosts under monitoring break; case 30: + $scope.$apply(function(){ + $scope.state = value; + }); toastr.info(value, "Current status is :"); /* $scope.$apply(function(){ diff --git a/static/js/controllers/table_ctrl.js b/static/js/controllers/table_ctrl.js index b515a37..bc8d1f6 100644 --- a/static/js/controllers/table_ctrl.js +++ b/static/js/controllers/table_ctrl.js @@ -22,120 +22,76 @@ tablemodule.controller('ctrlRead', function ($scope, $filter) { $scope.status_filter = ''; $scope.group_filter = ''; - $scope.items = [ - { - "addr":"192.168.74.1", - "name":"www.example.com", - "status":"success", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup1" - } - ], - "last_check":"14:51" - }, - { - "addr":"192.168.74.2", - "name":"www.example.com", - "status":"success", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup1" - } - ], - "last_check":"14:48" - }, - { - "addr":"192.168.74.3", - "name":"www.example.com", - "status":"success", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup1" - } - ], - "last_check":"14:51" - }, - { - "addr":"142.42.13.37", - "name":"www.nerd.org", - "status":"warning", - "group":[ - { - "name":"all" - } - ], - "last_check":"08:24" - }, - { - "addr":"135.47.86.11", - "name":"www.blabla.fr", - "status":"danger", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup2" - } - ], - "last_check":"12:18" - }, - { - "addr":"135.47.86.12", - "name":"www.blabla.fr", - "status":"success", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup2" - } - ], - "last_check":"12:20" - }, - { - "addr":"192.147.0.0", - "name":"", - "status":"warning", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup3" - } - ], - "last_check":"11:11" - }, - { - "addr":"192.147.0.1", - "name":"", - "status":"danger", - "group":[ - { - "name":"all" - }, - { - "name":"mygroup3" - } - ], - "last_check":"11:14" - } - -]; + $scope.items = []; + + $scope.status = ''; + $scope.grp = "all"; + var ws = new WebSocket("ws://0.0.0.0:1337/websocket"); + + // actions effectuees lors de la reception d'un message via la websocket + ws.onmessage = function (evt) { + JSON.parse(evt.data, function (key, value) { + switch(parseInt(key)){ + case 20: // Success of a module execution + toastr.success(value, "Success on module execution"); + case 21: // Informations concerning one host + break; + case 22: // List of hosts under monitoring + $scope.$apply(function(){ + $scope.items = JSON.parse(value); + }); + break; + case 30: + $scope.$apply(function(){ + $scope.status = value; + }); + toastr.info(value, "Current status is :"); + /* + $scope.$apply(function(){ + $scope.state = value + });*/ + break; + case 31: + params = value.split(','); + if(params[0]=='success'){ + $scope.pop_success("Success on " + params[1], params[2]); + } + else if(params[0] == 'warning'){ + $scope.pop_warning("Warning on " + params[1], params[2]); + } + else if(params[0] == 'danger'){ + $scope.pop_danger("Danger on "+ params[1], params[2]); + } + break; + case 40: + toastr.error(value, "Server error"); + break; + default: + break; + } + }); + + }; + + $scope.pop_success = function(title, msg){ + toastr.success(msg, title); + }; + + $scope.pop_warning = function(title, msg){ + toastr.success(msg, title); + }; + + $scope.pop_danger = function(title, msg){ + toastr.error(msg, title); + }; + + $scope.getHosts = function(){ + var request = '{"14" : ""}'; + ws.send(request); + } + var searchMatch = function (haystack, needle) { if (!needle) { return true; @@ -263,7 +219,6 @@ tablemodule.controller('ctrlRead', function ($scope, $filter) { item.Selected = $scope.selectedAll; }); }; - }); diff --git a/views/dashboard.html b/views/dashboard.html index 8c6f056..9995f95 100644 --- a/views/dashboard.html +++ b/views/dashboard.html @@ -134,7 +134,7 @@ <pre>currentPage: {{currentPage|json}}</pre> <pre>currentPage: {{sort|json}}</pre>--> <tbody> - <tr ng-repeat="item in filteredItems | + <tr ng-repeat="item in items | orderBy:sort.sortingOrder:sort.reverse | filter:{addr:addr_filter, name:name_filter, status:status_filter, group:{name:group_filter}}" class={{item.status}}> @@ -146,6 +146,7 @@ </tr> </tbody> </table> + <button type="submit" class="btn btn-primary" ng-click="getHosts()">Get hosts</button> </div> </div> diff --git a/views/scan.html b/views/scan.html index bd2e3c7..f879936 100644 --- a/views/scan.html +++ b/views/scan.html @@ -62,11 +62,16 @@ </div> <div class="col-md-offset-2 main"> <h1 class="page-header">Scan for new machines</h1> - <form class="form-inline" ng_submit="post_val()"> - <label for="input_ip_range">IP range to scan (example : 198.116.0.1-10)</label> - <input type="ip_range" class="form-control" id="input_ip_range" ng-model="ip_range" placeholder="198.116.0.1-10"/> - <button type="submit" class="btn btn-primary">Scan now</button> - </form> + <div ng-show="validated == false" class="ng-hide"> + <form class="form-inline" ng_submit="post_val()"> + <label for="input_ip_range">IP range to scan (example : 198.116.0.1-10)</label> + <input type="ip_range" class="form-control" id="input_ip_range" ng-model="ip_range" placeholder="198.116.0.1-10"/> + <button type="submit" class="btn btn-primary" ng-click="validated = true">Scan now</button> + </form> + </div> + <div ng-show="validated == true"> + {{state}} + </div> </div> </div> </div> -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.