branch develop updated (4f1bea8 -> ad00240)
This is an automated email from the git hooks/post-receive script. New change to branch develop in repository mum. See http://git.chorem.org/mum.git from 4f1bea8 detection modules refactorés new ad00240 lock ajouté dans la classe shelve + refctoring des méthodes appellant process_monitoring afin de ne plus ajour de dépendances shelve/process_monitoring (tout centraliser dans module_loader) The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit ad00240170f3e0e9bd249b8908f6adb6b1e9de55 Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Fri Apr 3 12:00:26 2015 +0200 lock ajouté dans la classe shelve + refctoring des méthodes appellant process_monitoring afin de ne plus ajour de dépendances shelve/process_monitoring (tout centraliser dans module_loader) Summary of changes: app/module_loader.py | 43 ++++++++++++++++++++------- app/modules/storage_modules/shelve_db.py | 50 +++++++++++++++++--------------- app/mum.py | 21 ++++++-------- static/js/controllers/headCtrl.js | 3 ++ static/js/controllers/hostPageCtrl.js | 15 +++++----- 5 files changed, 79 insertions(+), 53 deletions(-) -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.
This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit ad00240170f3e0e9bd249b8908f6adb6b1e9de55 Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Fri Apr 3 12:00:26 2015 +0200 lock ajouté dans la classe shelve + refctoring des méthodes appellant process_monitoring afin de ne plus ajour de dépendances shelve/process_monitoring (tout centraliser dans module_loader) --- app/module_loader.py | 43 ++++++++++++++++++++------- app/modules/storage_modules/shelve_db.py | 50 +++++++++++++++++--------------- app/mum.py | 21 ++++++-------- static/js/controllers/headCtrl.js | 3 ++ static/js/controllers/hostPageCtrl.js | 15 +++++----- 5 files changed, 79 insertions(+), 53 deletions(-) diff --git a/app/module_loader.py b/app/module_loader.py index 1328a59..451aa08 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -8,6 +8,8 @@ import modules.storage_modules import modules.ModuleNotCompatibleException import modules.CommandNotFoundException import modules.HostNotFoundException +from modules.notification_modules.websocket_container import WebSocketContainer +import process_monitoring import json import re @@ -21,28 +23,25 @@ class ModuleLoader: Loads dynamically modules from packages connection_modules, detection_modules, monitoring_modules, storage_modules and contains several methods in order to call the methods once these modules loaded. """ - def __init__(self, add_func, rem_func, conf): + def __init__(self, conf): self.conf = conf - ''' - self.db_loc = dict_conf['db_location'] - self.external_mod_loc = dict_conf['external_modules_location'] - self.public_keys_loc = dict_conf['keys_location'] - ''' - self.db = self.load_db(add_func, rem_func, self.conf['keys_location']) + self.db = self.load_db(self.conf['keys_location']) self.loaded_mod_moni = {} # See load_all_monitoring_modules self.loaded_mod_detect = {} # See load_all_detection_modules self.loaded_mod_conn = {} # See load_all_connection_modules self.loaded_mod_notif = {} # See load_all_notification_modules self.compatible_os_list = ['other'] # Will contain the list of os compatibles for every monitoring module loaded + self.wsc = WebSocketContainer(self.db) + process_monitoring.init(self, self.wsc) - def load_db(self, add_func, rem_func, key_loc): + def load_db(self, key_loc): """ Creates an instance of the class shelve_db from storage_modules. :return: an instance of the shelve_db class """ db_name = "shelve_db" db = __import__("modules.storage_modules." + db_name, fromlist=modules.storage_modules) - db_instance = getattr(db, db_name)(self.conf['db_location'], add_func, rem_func, key_loc) + db_instance = getattr(db, db_name)(self.conf['db_location'], key_loc) return db_instance def get_db(self): @@ -51,6 +50,17 @@ class ModuleLoader: def get_public_keys_loc(self): return self.conf['keys_location'] + def get_websocket_container(self): + return self.wsc + + @staticmethod + def add_to_waiting_list(instr): + process_monitoring.add_to_waiting_list(instr) + + @staticmethod + def stop_monitoring(): + process_monitoring.end = True + @staticmethod def run_nmap_detection(param, opt, db, ws, list_mod_conn, dict_mod_monitoring): """ @@ -397,11 +407,24 @@ class ModuleLoader: return getattr(self.db, dict_instr['func'])(dict_instr['args']) def update_activated_modules(self, args): - self.db.config_mod_activation(args, self.get_monitoring_modules_list()) + dict_instr = self.db.config_mod_activation(args) + for add_instr in dict_instr['add']: + process_monitoring.add_to_waiting_list(add_instr) + for rem_instr in dict_instr['rem']: + process_monitoring.remove_to_waiting_list(rem_instr['addr'], rem_instr['mod_name']) + + def set_conf_moni_mod(self, args): + self.db.set_conf_mod(args) + process_monitoring.update_mod_on_waiting_list(args) def get_public_keys_list(self): return os.listdir(self.conf['keys_location']) + def remove_host(self, args): + self.db.remove_host(args) + # removing monitoring instructions for this host + process_monitoring.remove_to_waiting_list(args['addr_host'], None) + def get_host_info(self, addr_host): host_info = self.db.get_host_informations(addr_host) host_info['compatible_os_list'] = self.compatible_os_list diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index 36bae67..5aaa385 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -4,6 +4,7 @@ from datetime import datetime import json import shelve import traceback +import threading import os.path @@ -14,18 +15,19 @@ class shelve_db: Storage module for the persistant objects in Python : Shelve. Every function in need to access the database have to be moved in this class. """ - def __init__(self, db_loc, add_func, rem_func, key_loc): + def __init__(self, db_loc, key_loc): self.db = None + self.lock = threading.Lock() self.db_loc = db_loc - self.add_to_monitoring_list = add_func - self.rem_to_monitoring_list = rem_func self.key_loc = key_loc def open_db(self): """ Open the shelve database from the file specified in conf.txt. If the file donesn't exists, it will be created and the first structure will also be initialized. + Moreover, a lock is activated during all the transaction, because Shelve is not thread safe. """ + self.lock.acquire() if not os.path.isfile(self.db_loc): # init of the database at the first opening self.db = shelve.open(self.db_loc, writeback=True) try: @@ -40,10 +42,11 @@ class shelve_db: def close_db(self): """ - Closes the database + Closes the database and release the lock. """ self.db.close() self.db = None + self.lock.release() def init_global_conf(self, loaded_mod_moni): """ @@ -397,8 +400,6 @@ class shelve_db: addr_host = args['addr_host'] self.open_db() try: - # removing monitoring instructions for this host - self.rem_to_monitoring_list(addr_host, None) # removing monitoring entries for this host if addr_host in self.db['hosts']: del self.db['hosts'][addr_host] @@ -489,7 +490,7 @@ class shelve_db: finally: self.close_db() - def config_mod_activation(self, args, dict_mod_info): + def config_mod_activation(self, args): """ Activates or desactivates monitoring modules for a given host :param args: a dictionary containing : @@ -499,19 +500,16 @@ class shelve_db: mod_name: bool } } - :param dict_mod_info: A dictionary containing informations about all notification modules, in the form: + :return: a dictionary containing the instructions to send to the process monitoring in the form: { - mod_name: - { - 'class_name': string, => the name of the class to instanciate - 'compatible_os': [string1, string2, ...], => a list containing the compatibles os - 'unit': string, => the unit type of return ('%', 'bool' or other) - 'block': string, => the monitoring block of the module - 'external': bool => indicates if this modules comes from external directory - } + 'add': [{'addr': str, 'os': str, 'mod_name': str, 'time': datetime, 'freq': int}, ...] + 'rem': [{'addr': str, 'mod_name': str}, ... ] } """ addr_host = args['addr_host'] + dict_instr = {} + dict_instr['add'] = [] + dict_instr['rem'] = [] self.open_db() try: for mod_name in args['activated']: @@ -526,21 +524,25 @@ class shelve_db: if args['activated'][mod_name]: # if it is now activated, we have to add it to the monitoring list self.db["hosts"][addr_host]["conf"]["monitoring"][mod_name]["activated"] = True - dict_instr = {} - dict_instr['addr'] = addr_host - dict_instr['os'] = json.loads(self.db['hosts'][addr_host]['detected']['nmap'])['os'] - dict_instr['mod_name'] = mod_name - dict_instr['time'] = datetime.now() - dict_instr['freq'] = self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['check_frequency'] - self.add_to_monitoring_list(dict_instr) + add_instr = {} + add_instr['addr'] = addr_host + add_instr['os'] = json.loads(self.db['hosts'][addr_host]['detected']['nmap'])['os'] + add_instr['mod_name'] = mod_name + add_instr['time'] = datetime.now() + add_instr['freq'] = self.db['hosts'][addr_host]['conf']['monitoring'][mod_name]['check_frequency'] + dict_instr['add'].append(add_instr) else: # if not, we have to remove it from the monitoring list self.db["hosts"][addr_host]["conf"]["monitoring"][mod_name]["activated"] = False - self.rem_to_monitoring_list(args['addr_host'], mod_name) + rem_instr = {} + rem_instr['addr'] = args['addr_host'] + rem_instr['mod_name'] = mod_name + dict_instr['rem'].append(rem_instr) except Exception: print traceback.format_exc() finally: self.close_db() + return dict_instr def get_conf_mod(self, args): """ diff --git a/app/mum.py b/app/mum.py index fee7382..e84e63d 100755 --- a/app/mum.py +++ b/app/mum.py @@ -9,8 +9,6 @@ from geventwebsocket import WebSocketError import json import threading from module_loader import ModuleLoader -import process_monitoring -from modules.notification_modules.websocket_container import WebSocketContainer import argparse # Pour lancer la detection nmap avec un nouveau thread @@ -32,7 +30,7 @@ class ThreadDetect(threading.Thread): for ip in json.loads(scanned_ip): monitoring_intructions = db.get_monitoring_instructions(ip) for instr in monitoring_intructions: - process_monitoring.add_to_waiting_list(instr) + ml.add_to_waiting_list(instr) """ # now launching full detection for ip in json.loads(scanned_ip): @@ -117,7 +115,7 @@ def bower_files(filepath): @get('/websocket', apply=[websocket]) def receive(ws): global ml - wsc.add_websocket(ws) + ml.get_websocket_container().add_websocket(ws) while True: try: response = ws.receive() @@ -134,6 +132,9 @@ def receive(ws): ws.send(json.dumps({"RES_GET_HOSTS": db.get_hosts()})) elif code == "GET_HOST_INFO": # asked from hostpage ws.send(json.dumps({"RES_INFO_HOST": ml.get_host_info(msg["GET_HOST_INFO"])})) + elif code == "REM_HOST": + ml.remove_host(msg["REM_HOST"]) + ws.send(json.dumps({"RES_REM_HOST": True})) elif code == "CALL_FUNC_DB": # asked when the request can directly pass by database res = ml.launch_db_function(msg["CALL_FUNC_DB"]) msg = json.dumps({"RES_CALL_FUNC_DB": { @@ -159,8 +160,7 @@ def receive(ws): except Exception: ws.send(json_dumps({"RES_TEST_CONN": False})) elif code == "SET_CONF_MOD": # asked from hostpage, at the monitoring module configuration - ml.launch_db_function({'func': 'set_conf_mod', 'args': msg["SET_CONF_MOD"]}) - process_monitoring.update_mod_on_waiting_list(msg["SET_CONF_MOD"]) + ml.set_conf_moni_mod(msg["SET_CONF_MOD"]) elif code == "CHECK_NOW": # asekd from hostpage args = msg["CHECK_NOW"] ml.run_one_monitoring_module(args['mod_name'], args['addr_host'], None, None) @@ -169,7 +169,7 @@ def receive(ws): else: break except WebSocketError: # Should be WebSocketError when closing the connection - wsc.remove_websocket(ws) + ml.get_websocket_container().remove_websocket(ws) break @route('/upload', method='POST') @@ -186,7 +186,6 @@ def do_upload(): # Lancement du serveur a l'adresse 0.0.0.0:1337 if __name__ == '__main__': global ml - global wsc # creating the parser for the command line arguments parser = argparse.ArgumentParser(description="Mum (Machines Under Monotoring) is a web-oriented and modulable " @@ -219,17 +218,15 @@ if __name__ == '__main__': if dict_args[arg] is not None: conf[arg] = dict_args[arg] - ml = ModuleLoader(process_monitoring.add_to_waiting_list, process_monitoring.remove_to_waiting_list, conf) + ml = ModuleLoader(conf) ml.load_all_monitoring_modules() ml.load_all_connection_modules() ml.load_all_detection_modules() ml.load_all_notification_modules() ml.get_db().init_global_conf(ml.get_monitoring_modules_list()) - wsc = WebSocketContainer(ml.get_db()) #dict_notif = ml.db.add_check('127.0.0.1', "ping", False) #ml.run_notification_modules(dict_notif) - process_monitoring.init(ml, wsc) port = int(os.environ.get('PORT', int(conf['server_port']))) run(host=conf['server_addr'], port=port, debug=True, server=GeventWebSocketServer) # after ending - process_monitoring.end = True \ No newline at end of file + ml.stop_monitoring() \ No newline at end of file diff --git a/static/js/controllers/headCtrl.js b/static/js/controllers/headCtrl.js index eea029c..d57023e 100644 --- a/static/js/controllers/headCtrl.js +++ b/static/js/controllers/headCtrl.js @@ -41,6 +41,9 @@ mumApp.controller('headCtrl', function($scope, $rootScope, toastr, $interval, $r $scope.items = DataHosts.Items; }); break; + case "RES_REM_HOST": + $rootScope.$broadcast("remHost"); + break; case "RES_GET_LOADED_CONN_MOD": $rootScope.$broadcast("resGetLoadedConnMod", obj[key]); break; diff --git a/static/js/controllers/hostPageCtrl.js b/static/js/controllers/hostPageCtrl.js index 23f562e..a35f434 100644 --- a/static/js/controllers/hostPageCtrl.js +++ b/static/js/controllers/hostPageCtrl.js @@ -49,12 +49,6 @@ mumApp.controller('hostPageCtrl', function($scope, $rootScope, $route, $routePar if(args.func == 'update_custom_informations'){ $route.reload(); } - if(args.func == 'remove_host'){ - $scope.$apply(function(){ - $rootScope.$broadcast("sendViaWs", JSON.stringify({"GET_HOSTS": ""})); - $location.path('/'); - }); - } }); $scope.model = {custom_infos : '', @@ -100,6 +94,13 @@ mumApp.controller('hostPageCtrl', function($scope, $rootScope, $route, $routePar $rootScope.$broadcast("sendViaWs", JSON.stringify({"GET_HOST_INFO": $scope.addr_host})); }); + $scope.$on("remHost", function (event, args) { + $scope.$apply(function(){ + $rootScope.$broadcast("sendViaWs", JSON.stringify({"GET_HOSTS": ""})); + $location.path('/'); + }); + }); + // save custom informations $scope.save_custom_infos = function(){ var args = {}; @@ -111,7 +112,7 @@ mumApp.controller('hostPageCtrl', function($scope, $rootScope, $route, $routePar $scope.remove_host = function(){ var args = {}; args['addr_host'] = $scope.addr_host; - $rootScope.$broadcast("sendViaWs", JSON.stringify({"CALL_FUNC_DB": {'func': 'remove_host', 'args': args}})); + $rootScope.$broadcast("sendViaWs", JSON.stringify({"REM_HOST": args})); }; // creation of modals -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.
participants (1)
-
chorem.org scm