This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit 326868b9ee30255d8b3e0312d4a9ab578d50dca6 Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Fri Feb 20 15:51:19 2015 +0100 several functions from shelve implemented --- .gitignore | 1 + app/app.py | 8 +- app/module_loader.py | 27 ++- app/modules/detection_modules/nmap_detection.py | 7 +- .../monitoring_modules/unix/updated_packages.py | 11 +- app/modules/storage_modules/shelve_db.py | 214 +++++++++++++++++++-- 6 files changed, 239 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 6cae0cc..ef12ce2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ node_modules .idea res.xml mum.db +*.pyc \ No newline at end of file diff --git a/app/app.py b/app/app.py index abba633..a1336f5 100755 --- a/app/app.py +++ b/app/app.py @@ -7,7 +7,7 @@ from bottle_websocket import GeventWebSocketServer from bottle_websocket import websocket import json import threading -from module_loader import * +import module_loader # Pour lancer la detection nmap avec un nouveau thread @@ -18,8 +18,8 @@ class ThreadDetect(threading.Thread): self.ws = ws def run(self): - db = load_db() - scanned_ip = run_nmap_detection(self.ip_range, db, self.ws) + db = module_loader.load_db() + scanned_ip = module_loader.run_nmap_detection(self.ip_range, db, self.ws) self.ws.send(json.dumps({"20" : scanned_ip})) @@ -108,7 +108,7 @@ def receive(ws): if code == "10": start_first_detection(msg["10"], ws) elif code == "14": - db = load_db() + db = module_loader.load_db() ws.send(json.dumps({"22": db.get_hosts()})) del db else: diff --git a/app/module_loader.py b/app/module_loader.py index c336695..cea3d72 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -55,18 +55,39 @@ def run_all_detection_modules(os, conn, db): :param db: an instance of a storage module """ __import__("modules.detection_modules." + os) - pack_mod_os = __import__("modules.detection_modules." + os, fromlist=modules.detection_modules.__all__) + pack_mod_os = __import__("modules.detection_modules." + os, fromlist=modules.detection_modules.__all__) for mod_name in pack_mod_os.__all__: mod = __import__ ("modules.detection_modules." + os + "." + mod_name, fromlist=modules.detection_modules.unix.__all__) # on charge le module mod_instance = getattr(mod, mod_name)(conn, db) # on appelle le constructeur mod_instance.run_detection() - def run_all_monitoring_modules(os, conn, db): + """ + Instanciates and runs every monitoring_modules listed in the __init__.py file of the package corresponding to + the operating system entered in parameters. + :param os: the oprating system of the host + :param conn: an instance of a connection module + :param db: an instance of a storage module + """ __import__("modules.monitoring_modules." + os) pack_mod_os = __import__("modules.monitoring_modules." + os, fromlist=modules.monitoring_modules.__all__) for mod_name in pack_mod_os.__all__: mod = __import__ ("modules.monitoring_modules." + os + "." + mod_name, fromlist=modules.monitoring_modules.unix.__all__) # on charge le module mod_instance = getattr(mod, mod_name)(conn, db) # on appelle le constructeur - #mod_instance.check() + mod_instance.check() + + +def run_one_monitoring_module(mod_name, os, conn, db): + """ + Instanciates and runs one monitoring_module of the package corresponding to + the operating system entered in parameters. + :param mod_name: the name of the monitoring_module to run + :param os: the oprating system of the host + :param conn: an instance of a connection module + :param db: an instance of a storage module + """ + __import__("modules.monitoring_modules." + os) + mod = __import__("modules.monitoring_modules." + os + "." + mod_name, fromlist=modules.monitoring_modules.unix.__all__) + mod_instance = getattr(mod, mod_name)(conn, db) # on appelle le constructeur + mod_instance.check() \ No newline at end of file diff --git a/app/modules/detection_modules/nmap_detection.py b/app/modules/detection_modules/nmap_detection.py index 9fe5a3a..f8c379e 100644 --- a/app/modules/detection_modules/nmap_detection.py +++ b/app/modules/detection_modules/nmap_detection.py @@ -81,7 +81,7 @@ class nmap_detection: ip = str(byte_1) + '.' + str(byte_2) + '.' + str(byte_3) + '.' + str(byte_4) self.ws.send(json.dumps({"30": "Scanning ip : " + ip})) try: - child = pexpect.spawn('nmap', ['-A', ip, '-oX', 'res.xml']) + child = pexpect.spawn('nmap', ['-A', '-Pn', ip, '-oX', 'res.xml']) while child.isalive(): child.expect('Completed', timeout=None) except pexpect.EOF: @@ -105,6 +105,10 @@ class nmap_detection: # get every <host> of the collection hosts = collection.getElementsByTagName("host") + # if host cannot have been detected, adds it empty + if hosts == []: + self.db.add_host(ip, {}) + # Get the nodes of each <host> and recuperaton of their attributes # JSON = dictionary list for host in hosts: @@ -137,6 +141,7 @@ class nmap_detection: list_dict_port.append(dict_port) dict_host['openports'] = list_dict_port # the host have its IP for ID on the db + print dict_host self.db.add_host(dict_host['addr'], json.dumps(dict_host)) pexpect.run("rm -f res.xml") self.scanned_ip.append(ip) \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/updated_packages.py b/app/modules/monitoring_modules/unix/updated_packages.py index 4a5232b..9d97802 100644 --- a/app/modules/monitoring_modules/unix/updated_packages.py +++ b/app/modules/monitoring_modules/unix/updated_packages.py @@ -6,7 +6,8 @@ class updated_packages: def __init__(self, conn, db): self.conn = conn self.db = db - self.part = "software" + self.block = "software" + self.unit = "bool" def check(self): cmd = "apt-get upgrade -s" @@ -16,4 +17,10 @@ class updated_packages: res_check = json.dumps({'non_updated_packages': False}) else: res_check = json.dumps({'non_updated_packages': True}) - self.db.add_check(self.conn.get_addr_host(), "updated_packages", res_check) \ No newline at end of file + self.db.add_check(self.conn.get_addr_host(), "updated_packages", res_check) + + def get_block(self): + return self.block + + def get_unit(self): + return self.unit \ No newline at end of file diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index 74282b7..418f363 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -40,11 +40,10 @@ class shelve_db: self.db.close() self.db = None - # Add and save a new host after its first nmap detection # It also preconfigure with the default configuration, add the host to the group "all" and # creates empty structures for the monitoring and archive data. - def add_host(self,addr_host, nmap_res): + def add_host(self, addr_host, nmap_res): """ Called by the nmap_detection module. Add and save a new host after its first nmap detection @@ -63,10 +62,11 @@ class shelve_db: # Preconfiguration self.db["hosts"][addr_host]["conf"] = {} self.db["hosts"][addr_host]["conf"]["monitoring"] = self.db["global_conf"] - self.db["hosts"][addr_host]["conf"]["groups"] = {"name": "all"} + self.db["hosts"][addr_host]["conf"]["groups"] = ["all"] # Every host is in group "all" + self.db["hosts"][addr_host]["conf"]["connections"] = {} self.db["hosts"][addr_host]["conf"]["subscribers"] = {} # Add current user automatically ? self.db["hosts"][addr_host]["conf"]["custom_info"] = "" - self.db["hosts"][addr_host]["conf"]["interventions"] = {} + self.db["hosts"][addr_host]["conf"]["interventions"] = [] # Create structure for monitoring data self.db["hosts"][addr_host]["monitoring"] = {} # Create structure for archiving data @@ -74,7 +74,6 @@ class shelve_db: finally: self.close_db() - # Returns the essential data about all hosts under monitoring # These are used by the front-end # If no hosts have been added, the function will return an empty list @@ -88,8 +87,8 @@ class shelve_db: { "addr":"192.168.74.1", "name":"www.example.com", - "status":val, //"success" ou "warning" ou "danger" ou "" - "group":[ // au moins 1 groupe "all" + "status":val, //"success" or "warning" or "danger" or "" + "group":[ // at least 1 group "all" { "name":"all" }, @@ -97,7 +96,7 @@ class shelve_db: "name":"mygroup1" } ], - "last_check":val //heure UNIX + "last_check":val //UNIX time "subscribers":{ "uid":val, "priority":val @@ -122,7 +121,7 @@ class shelve_db: info_host["status"] = "" info_host["group"] = [] for group in self.db["hosts"][host]["conf"]["groups"]: - info_host["group"].append({"name": self.db["hosts"][host]["conf"]["groups"][group]}) + info_host["group"].append({"name": group}) if "date" in self.db["hosts"][host]["monitoring"]: info_host["last_check"] = self.db["hosts"][host]["monitoring"]["date"] else: @@ -132,6 +131,25 @@ class shelve_db: self.close_db() return json.dumps(res) + def remove_host(self, addr_host): + """ + Removes from the database the host at the adress on parameter. If the host is part of a group, it will be + removed of this group as well. + :param addr_host: the IP address of the host to delete + """ + self.open_db() + try: + # removing monitoring entries for this host + if addr_host in self.db['hosts']: + del self.db['hosts'][addr_host] + # removing this host for each group it is registered in + for group_id in self.db['groups']: + for host in self.db['groups'][group_id]['hosts']: + if self.db['groups'][group_id]['hosts'][host]['addr'] == addr_host: + del self.db['groups'][group_id]['hosts'][host] + finally: + self.close_db() + def save_detection(self, addr_host, name_part, json_res_str): """ Called by a detection module in order to save his detection on the database. @@ -145,30 +163,77 @@ class shelve_db: finally: self.close_db() - def add_check(self, addr_host, name_part, val): + def update_custom_informations(self, addr_host, txt): + """ + Updates the custom informations stored on the host's configuration + :param addr_host: the IP adress of the host + :param txt: the new text to put on custom informations + """ + self.open_db() + try: + self.db["hosts"][addr_host]["conf"]["custom_info"] = txt + finally: + self.close_db() + + def add_intervention(self, addr_host, date, person, details): + """ + Add a new intervention, stored on the host's configuration + :param addr_host: the IP adress of the host + :param date: the intervention date + :param person: the username responsible of the intervention + :param details: a string explaining the intervetions + """ + intervention = {} + intervention['username'] = person + intervention['date'] = date + intervention['details'] = details + self.open_db() + try: + self.db["hosts"][addr_host]["conf"]["interventions"].append(intervention) + finally: + self.close_db() + + def config_mod_activation(self, addr_host, data_list): + """ + Activates or desactivates monitoring modules for a given host + :param addr_host: the IP adress of the host + :param data_list: the activation data on the form : + [ + {mod_name : bool}, + ... + ] + """ + self.open_db() + try: + for mod_name in data_list: + self.db["hosts"][addr_host]["conf"]["monitoring"][mod_name]["activated"] = data_list[mod_name] + finally: + self.close_db() + + def add_check(self, addr_host, mod_name, val): """ Called by a monitoring module. Add a new check of a host from a specific module. Add the previous entry of monitoring to the archive and call update_stats to update the statistics. :param addr_host: the IP adress of the host checked - :param name_part: the name of the monitoring_module which have done the check + :param mod_name: the name of the monitoring_module which have done the check :param val: the value observed """ self.open_db() new_val = {"date": datetime.now()} try: - if val >= self.db['hosts']['conf']['monitorig'][name_part]['minor_limit']: + if val >= self.db['hosts']['conf']['monitorig'][mod_name]['minor_limit']: new_val['state'] = 'warning' - elif val >= self.db['hosts']['conf']['monitorig'][name_part]['major_limit']: + elif val >= self.db['hosts']['conf']['monitorig'][mod_name]['major_limit']: new_val['state'] = 'danger' else: new_val['state'] = 'success' - previous_val = self.db['hosts'][addr_host]["monitoring"][name_part] - self.db['hosts'][addr_host]['monitoring'][name_part] = new_val + previous_val = self.db['hosts'][addr_host]["monitoring"][mod_name] + self.db['hosts'][addr_host]['monitoring'][mod_name] = new_val # now performing archiving - if self.db['hosts'][addr_host]['archive'].has_key(name_part): - self.db['hosts'][addr_host]['archive'][name_part] = \ - self.update_stats(self.db['hosts'][addr_host]['archive'][name_part], val) + if mod_name in self.db['hosts'][addr_host]['archive']: + self.db['hosts'][addr_host]['archive'][mod_name] = \ + self.update_stats(self.db['hosts'][addr_host]['archive'][mod_name], val) finally: self.close_db() @@ -191,4 +256,115 @@ class shelve_db: stats['delta'] = val - stats['mean'] stats['mean'] += stats['delta'] / stats['nb_check'] stats['M2'] += stats['delta'] * (val - stats['mean']) - return stats \ No newline at end of file + return stats + + def create_global_conf(self, dict): + """ + Create an entry on the global_conf for each new monitoring module. + :param dict: dictionary containing informations about all notification modules, by os, in the form: + [os_name][monitoring_module_name][{'block':val, 'unit': val}] + """ + self.open_db() + try: + for os in dict: + if os not in self.db['global_conf']: + self.db['global_conf'][os] = {} + for mod in dict[os]: + if mod not in self.db['global_conf'][os]: + self.db['global_conf'][os][mod] = {} + self.db['global_conf'][os][mod]['block'] = dict[os][mod]['block'] + self.db['global_conf'][os][mod]['activated'] = True + self.db['global_conf'][os][mod]['check_frequency'] = 3600 + self.db['global_conf'][os][mod]['nb_minute'] = 30 + self.db['global_conf'][os][mod]['nb_hour'] = 12 + self.db['global_conf'][os][mod]['nb_day'] = 15 + self.db['global_conf'][os][mod]['nb_week'] = 2 + self.db['global_conf'][os][mod]['nb_month'] = 6 + self.db['global_conf'][os][mod]['nb_year'] = None + unit = dict[os][mod]['unit'] + self.db['global_conf'][os][mod]['unit'] = unit + if unit == '%': + self.db['global_conf'][os][mod]['minor_limit'] = 95 + self.db['global_conf'][os][mod]['major_limit'] = 100 + elif unit == 'bool': + self.db['global_conf'][os][mod]['minor_limit'] = False + self.db['global_conf'][os][mod]['major_limit'] = False + else: + self.db['global_conf'][os][mod]['minor_limit'] = 8 + self.db['global_conf'][os][mod]['major_limit'] = 10 + finally: + self.close_db() + + def add_host_list_to_group(self, host_list, group): + """ + Add given hosts to a group. If the group doesn't exists on the database, it will be created. + :param host_list: a list of IP adresses + :param group: the name of the group + """ + self.open_db() + try: + # if the group is not registered, we create it + if group not in self.db['groups']: + self.db['groups'][group] = {} + self.db['groups'][group]['hosts'] = [] + self.db['groups'][group]['subscribers'] = [] + for host in host_list: + self.db['groups'][group].append(host) + self.db['hosts'][host]['conf']['groups'].append(group) + finally: + self.close_db() + + def remove_host_list_to_group(self, host_list, group): + """ + Remove given hosts to a group. If the group is empty afterwards, il will be also deleted. + :param host_list: a list of IP adresses + :param group: the name of the group + """ + self.open_db() + try: + for host in host_list: + self.db['hosts'][host]['conf']['groups'].remove(group) + self.db['groups'][group]['hosts'].remove(host) + # deletion of the group if empty + if len(self.db['groups'][group]['hosts']) == 0: + del self.db['groups'][group] + finally: + self.close_db() + + def create_user(self, username): + """ + Create a basic empty structure on the database for a new user. + :param username:string containing the name of the user + """ + self.open_db() + try: + self.db['users'][username] = {} + self.db['users'][username]['preferences'] = {} + self.db['users'][username]['preferences']['minor_notifications'] = {} + self.db['users'][username]['preferences']['major_notifications'] = {} + self.db['users'][username]['account'] = {} + finally: + self.close_db() + + def remove_user(self, username): + """ + Removes a user from the database. If the user is registered to a host or a group, it is also deleted from these lists + :param username: string containing the name of the user + """ + self.open_db() + try: + # deletion of the user + del self.db['users'][username] + # unregistering user from hosts + for host in self.db['hosts']: + for subscriber in self.db['hosts'][host]['conf']['subscribers']: + if username in subscriber['username']: + self.db['hosts'][host]['conf']['subscribers'].remove(subscriber) + # unregistering user form groups + for group in self.db['groups']: + for subscriber in self.db['groups'][group]['subscribers']: + if username in subscriber['username']: + self.db['groups'][group]['subscribers'].remove(subscriber) + finally: + self.close_db() + -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.

This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit 9d8f7dd70ae45432e1b0935f7a60d6b22c130cbc Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Fri Feb 20 18:25:54 2015 +0100 memory module added --- app/module_loader.py | 2 +- app/modules/connection_modules/ssh.py | 30 +--------------------- app/modules/monitoring_modules/unix/__init__.py | 5 ++-- app/modules/monitoring_modules/unix/memory.py | 22 ++++++++++++++++ .../monitoring_modules/unix/updated_packages.py | 10 +------- app/modules/storage_modules/shelve_db.py | 4 +-- 6 files changed, 30 insertions(+), 43 deletions(-) diff --git a/app/module_loader.py b/app/module_loader.py index d084005..8ea04d2 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -104,7 +104,7 @@ def get_info_mod_monitoring(os): } """ __import__("modules.monitoring_modules." + os) - pack_mod_os = __import__("modules.monitoring_modules." + os, fromlist=modules.monitoring_modules.__all__) + pack_mod_os = __import__("modules.monitoring_modules." + os, fromlist=modules.monitoring_modules.__all__) return pack_mod_os.info_mod diff --git a/app/modules/connection_modules/ssh.py b/app/modules/connection_modules/ssh.py index 9befed0..0d799b9 100644 --- a/app/modules/connection_modules/ssh.py +++ b/app/modules/connection_modules/ssh.py @@ -19,32 +19,4 @@ class ssh: return res def disconnect(self): - self.ssh.close() - - - ''' - # Informations materielles globales - def detect_hardware(ssh): - cmd = "lshw -json" - stdin, stdout, stderr = ssh.exec_command(cmd) - res = "" - for line in stdout.read().splitlines(): - res += line - res_json = json.loads(res) - # TODO Traitement du resultat pour garder l'essentiel... - return res_json - - - - - # dependant de la langue du systeme ? - def detect_non_updated_packages(ssh): - cmd = "apt-get upgrade -s" - stdin, stdout, stderr = ssh.exec_command(cmd) - res = stdout.read() - tab_res = res.split(':') - if len(tab_res) == 2: - return json.dumps({'non_updated_packages': False}) - else: - return json.dumps({'non_updated_packages': True}) - ''' \ No newline at end of file + self.ssh.close() \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/__init__.py b/app/modules/monitoring_modules/unix/__init__.py index ac9e130..8efe043 100644 --- a/app/modules/monitoring_modules/unix/__init__.py +++ b/app/modules/monitoring_modules/unix/__init__.py @@ -1,5 +1,6 @@ __author__ = 'aguilbaud' -__all__=['updated_packages'] +__all__=['updated_packages', 'memory'] info_mod = { - 'updated_packages': {'block': 'software', 'unit': 'bool'} + 'updated_packages': {'block': 'software', 'unit': 'bool'}, + 'memory': {'block': 'hardware', 'unit': 'kB'} } \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/memory.py b/app/modules/monitoring_modules/unix/memory.py new file mode 100644 index 0000000..2b36cf5 --- /dev/null +++ b/app/modules/monitoring_modules/unix/memory.py @@ -0,0 +1,22 @@ +__author__ = 'aguilbaud' +import json +import re + +class memory: + def __init__(self, conn, db): + self.conn = conn + self.db = db + + def check(self): + cmd = "cat /proc/meminfo" + stdout = self.conn.exec_command(cmd) + memfree = 0 + memtotal = 0 + for line in stdout.splitlines(): + tab_res = line.split(':') + if(tab_res[0]) == 'MemTotal': + memtotal = re.sub("[^0-9]", "", tab_res[1]) + elif(tab_res[0]) == 'MemFree': + memfree = re.sub("[^0-9]", "", tab_res[1]) + res_check = json.dumps({"memory": (int(memfree) * 100) / int(memtotal)}) + self.db.add_check(self.conn.get_addr_host(), "memory", res_check) \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/updated_packages.py b/app/modules/monitoring_modules/unix/updated_packages.py index 9d97802..a0642fc 100644 --- a/app/modules/monitoring_modules/unix/updated_packages.py +++ b/app/modules/monitoring_modules/unix/updated_packages.py @@ -6,8 +6,6 @@ class updated_packages: def __init__(self, conn, db): self.conn = conn self.db = db - self.block = "software" - self.unit = "bool" def check(self): cmd = "apt-get upgrade -s" @@ -17,10 +15,4 @@ class updated_packages: res_check = json.dumps({'non_updated_packages': False}) else: res_check = json.dumps({'non_updated_packages': True}) - self.db.add_check(self.conn.get_addr_host(), "updated_packages", res_check) - - def get_block(self): - return self.block - - def get_unit(self): - return self.unit \ No newline at end of file + self.db.add_check(self.conn.get_addr_host(), "updated_packages", res_check) \ No newline at end of file diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index d04297c..b3c2d7e 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -287,8 +287,8 @@ class shelve_db: self.db['global_conf'][os][mod]['minor_limit'] = 95 self.db['global_conf'][os][mod]['major_limit'] = 100 elif unit == 'bool': - self.db['global_conf'][os][mod]['minor_limit'] = False - self.db['global_conf'][os][mod]['major_limit'] = True + self.db['global_conf'][os][mod]['minor_limit'] = True + self.db['global_conf'][os][mod]['major_limit'] = False else: self.db['global_conf'][os][mod]['minor_limit'] = 8 self.db['global_conf'][os][mod]['major_limit'] = 10 -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.