This is an automated email from the git hooks/post-receive script. New commit to branch develop in repository mum. See http://git.chorem.org/mum.git commit 492e5c18d0dc2e8f13c7259f28707532285d9f8a Author: Alexis Guilbaud <guilbaud@codelutin.com> Date: Fri Mar 6 16:57:39 2015 +0100 Les modules de monitoring sont maintenant chargés : d'une part ceux internes, d'autre part ceux dans le dossier extérieur spécifiés dans conf.txt --- app/app.py | 1 + app/module_loader.py | 110 +++++++++++++-------- app/modules/monitoring_modules/__init__.py | 1 - app/modules/monitoring_modules/{unix => }/disk.py | 24 ++++- .../monitoring_modules/{unix => }/memory.py | 22 ++++- app/modules/monitoring_modules/ping.py | 23 ++++- app/modules/monitoring_modules/unix/__init__.py | 7 -- .../monitoring_modules/unix/updated_packages.py | 16 --- app/modules/monitoring_modules/updated_packages.py | 37 +++++++ app/modules/storage_modules/shelve_db.py | 21 ++-- app/process_monitoring.py | 5 +- conf.txt | 2 +- 12 files changed, 188 insertions(+), 81 deletions(-) diff --git a/app/app.py b/app/app.py index 038f406..81a94f3 100755 --- a/app/app.py +++ b/app/app.py @@ -168,6 +168,7 @@ if __name__ == '__main__': global ml global wsc ml = ModuleLoader() + ml.get_all_monitoring_modules() wsc = WebSocketContainer(ml.get_db()) process_monitoring.init(ml, wsc) port = int(os.environ.get('PORT', 1337)) diff --git a/app/module_loader.py b/app/module_loader.py index 4a00f9a..4abb83e 100644 --- a/app/module_loader.py +++ b/app/module_loader.py @@ -1,4 +1,5 @@ __author__ = 'aguilbaud' + import modules import modules.connection_modules import modules.detection_modules @@ -26,6 +27,7 @@ class ModuleLoader: self.db_loc = dict_conf['db_location'] self.external_mod_loc = dict_conf['external_modules_location'] self.db = self.load_db() + self.loaded_mod_infos = {} def load_db(self): """ @@ -62,7 +64,6 @@ class ModuleLoader: ws.send(json.dumps({"40": hnfe.__str__()})) return None - def load_conn(self, conn_name, addr_host, username, key_location): # /home/aguilbaud/.ssh/id_rsa """ Instanciates and creates a connection with a connection module. @@ -75,7 +76,6 @@ class ModuleLoader: conn_instance = getattr(conn, conn_name)(addr_host, username, key_location, modules.CommandNotFoundException) return conn_instance - def run_all_detection_modules(self, os, conn, db, ws): """ Instanciates and runs every detection_modules listed in the __init__.py file of the package corresponding to @@ -97,22 +97,46 @@ class ModuleLoader: if ws is not None: ws.send(json.dumps({"40": cnfe.__str__()})) - - def run_all_monitoring_modules(self):#os, conn, db, ws): + def get_all_monitoring_modules(self): """ - Instanciates and runs every monitoring_modules listed in the __init__.py file of the package corresponding to - the operating system entered in parameters. - :param os: the oprating system of the host - :param conn: an instance of a connection module - :param db: an instance of a storage module - :param ws: a websocket connection if the function have been called from a client. Is None otherwise + Instanciates and stores the informations about each monitoring modules avaliable (internal and loaded + externally) """ - mod_list = [] - for importer, package_name, _ in pkgutil.iter_modules(["modules/monitoring_modules/"]): - full_package_name = "modules/monitoring_modules." + package_name - mod = importer.find_module(package_name).load_module(full_package_name) - mod_list.append(mod) - return mod_list + for importer, mod_name, ispkg in pkgutil.iter_modules(["app/modules/monitoring_modules/"]): + if mod_name not in sys.modules: + loaded_mod = __import__("modules.monitoring_modules." + mod_name, fromlist=[mod_name]) + class_name = getattr(loaded_mod, "get_class_name")() + if mod_name == 'ping': + mod_inst = getattr(loaded_mod, class_name)(None, None) + else: + mod_inst = getattr(loaded_mod, class_name)(None, None, None) + infos_mod = {} + infos_mod['imported'] = loaded_mod + infos_mod['class_name'] = getattr(mod_inst, 'get_name')() + infos_mod['compatible_os'] = getattr(mod_inst, 'get_compatible_os')() + infos_mod['block'] = getattr(mod_inst, 'get_block')() + infos_mod['unit'] = getattr(mod_inst, 'get_unit')() + infos_mod['external'] = False + self.loaded_mod_infos[mod_name] = infos_mod + + # Now for external modules + # Adding the external diretory into pythonpath : + sys.path.insert(0, self.external_mod_loc) + for importer, mod_name, ispkg in pkgutil.iter_modules([self.external_mod_loc]): + if mod_name not in sys.modules: + loaded_mod = __import__(mod_name, fromlist=[mod_name]) + class_name = getattr(loaded_mod, "get_class_name")() + mod_inst = getattr(loaded_mod, class_name)(None, None, None) + infos_mod = {} + infos_mod['imported'] = loaded_mod + infos_mod['class_name'] = getattr(mod_inst, 'get_name')() + infos_mod['compatible_os'] = getattr(mod_inst, 'get_compatible_os')() + infos_mod['block'] = getattr(mod_inst, 'get_block')() + infos_mod['unit'] = getattr(mod_inst, 'get_unit')() + infos_mod['external'] = True + self.loaded_mod_infos[mod_name] = infos_mod + + """ __import__("modules.monitoring_modules." + os) @@ -132,26 +156,33 @@ class ModuleLoader: ws.send(json.dumps({"40": cnfe.__str__()})) """ - - def run_one_monitoring_module(self, mod_name, addr_host, os, conn, db, ws): + def run_one_monitoring_module(self, mod_name, addr_host, conn, db): """ Instanciates and runs one monitoring_module of the package corresponding to the operating system entered in parameters. :param mod_name: the name of the monitoring_module to run - :param os: the oprating system of the host :param conn: an instance of a connection module :param db: an instance of a storage module :param ws: a websocket connection if the function have been called from a client. Is None otherwise """ if db is None: db = self.get_db() + if mod_name == 'ping': + mod_inst = getattr(self.loaded_mod_infos[mod_name]['imported'], + self.loaded_mod_infos[mod_name]['class_name'])(db, addr_host) + mod_inst.check() + else: + mod_inst = getattr(self.loaded_mod_infos[mod_name]['imported'], + self.loaded_mod_infos[mod_name]['class_name'])(conn, db, + modules.ModuleNotCompatibleException) + mod_inst.check() + """ if conn is None: - if os is None and mod_name == 'ping': + if mod_name == 'ping': # in this case, the connection have not been configurated yet, so we just launch a ping check ping_mod = __import__("modules.monitoring_modules.ping", fromlist=modules.monitoring_modules.special_modules) ping_mod_inst = getattr(ping_mod, "ping")(db, addr_host) ping_mod_inst.check() - """ else: conn = load_conn("ssh", addr_host, "aguilbaud", "/home/aguilbaud/.ssh/id_rsa") __import__("modules.monitoring_modules." + os) @@ -169,23 +200,24 @@ class ModuleLoader: ws.send(json.dumps({"40": cnfe.__str__()})) """ - def get_info_mod_monitoring(self): """ - Get information about the output and block of the monitoring modules. These informations must be written by the - module developper on the __init__.py file (add on info_mod dictionnary). + Get information about the output, block, compatible os and class name of the monitoring modules. + These informations must be specified on each modules, and are loaded at the launch of the application with + the :return: a dictionary containing these informations on the form : - {'os:{ - mod_name: {'block': val, 'unit': 'bool' or '%' or unit_name} + { + mod_name: + { + 'class_name': val, => the name of the class to instanciate + 'compatible_os': [val1, val2, ...], => a list containing the compatibles os + 'unit': val, => the unit type of return ('%', 'bool' or other) + 'block': val, => the monitoring block of the module + 'external': False => indicates if this modules comes from external directory } } """ - info_mods = {} - for os in modules.monitoring_modules.__all__: - package = __import__("modules.monitoring_modules." + os, fromlist=modules.monitoring_modules.__all__) - info_mods[os] = package.info_mod - return info_mods - + return self.loaded_mod_infos def get_conection_modules_list(self): """ @@ -197,17 +229,12 @@ class ModuleLoader: pack_conn_os = __import__("modules.connection_modules", fromlist=modules.connection_modules.__all__) return pack_conn_os.__all__ - - def create_global_conf(self, db): + def update_global_conf(self): """ - Asks the database to create a global configuration in function of the monitoring modules descibed on the __init__.py - :param db: the database instance + Asks the database to update the configuration if new monitoring modules are added in function of the + modules informations stored. """ - dict_mod = {} - for os in modules.monitoring_modules.__all__: - dict_mod[os] = self.get_info_mod_monitoring() - db.create_global_conf(dict_mod) - + self.db.create_global_conf(self.loaded_mod_infos) def get_all_monitoring_instructions(self, db): """ @@ -221,7 +248,6 @@ class ModuleLoader: res.append(instr) return res - def launch_db_function(self, dict_instr): """ Calls a function of the database with the parametters sent. Used to each function which is not module dependant : diff --git a/app/modules/monitoring_modules/__init__.py b/app/modules/monitoring_modules/__init__.py index aa0ac17..30c04b0 100644 --- a/app/modules/monitoring_modules/__init__.py +++ b/app/modules/monitoring_modules/__init__.py @@ -1,3 +1,2 @@ __author__ = 'aguilbaud' -__all__ = ['unix'] special_modules = ['ping'] \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/disk.py b/app/modules/monitoring_modules/disk.py similarity index 67% rename from app/modules/monitoring_modules/unix/disk.py rename to app/modules/monitoring_modules/disk.py index c6f3d28..57b3ba7 100644 --- a/app/modules/monitoring_modules/unix/disk.py +++ b/app/modules/monitoring_modules/disk.py @@ -7,12 +7,32 @@ Check and returns the percentage of disk used ( = percentage of use of the parti """ -class disk: +def get_class_name(): + return "Disk" + + +class Disk: def __init__(self, conn, db, mnce): self.conn = conn self.db = db + self.name = get_class_name() + self.compatible_os = ['linux', 'unix'] + self.block = "hardware" + self.unit = '%' self.ModuleNotCompatibleException = mnce + def get_name(self): + return self.name + + def get_compatible_os(self): + return self.compatible_os + + def get_block(self): + return self.block + + def get_unit(self): + return self.unit + def check(self): cmd = "df -h" stdout = self.conn.exec_command(cmd) @@ -32,4 +52,4 @@ class disk: ) raise exception_inst res_check = int(disk_used) - self.db.add_check(self.conn.get_addr_host(), "disk", res_check) \ No newline at end of file + self.db.add_check(self.conn.get_addr_host(), self.name, res_check) \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/memory.py b/app/modules/monitoring_modules/memory.py similarity index 71% rename from app/modules/monitoring_modules/unix/memory.py rename to app/modules/monitoring_modules/memory.py index 1c2f146..5c7f4cf 100644 --- a/app/modules/monitoring_modules/unix/memory.py +++ b/app/modules/monitoring_modules/memory.py @@ -8,12 +8,32 @@ Check and returns the percentage of total memory used of the machine """ -class memory: +def get_class_name(): + return "Memory" + + +class Memory: def __init__(self, conn, db, mnce): self.conn = conn self.db = db + self.name = get_class_name() + self.compatible_os = ['linux', 'unix'] + self.block = "hardware" + self.unit = '%' self.ModuleNotCompatibleException = mnce + def get_name(self): + return self.name + + def get_compatible_os(self): + return self.compatible_os + + def get_block(self): + return self.block + + def get_unit(self): + return self.unit + def check(self): cmd = "cat /proc/meminfo" stdout = self.conn.exec_command(cmd) diff --git a/app/modules/monitoring_modules/ping.py b/app/modules/monitoring_modules/ping.py index de8f68b..f7467a8 100644 --- a/app/modules/monitoring_modules/ping.py +++ b/app/modules/monitoring_modules/ping.py @@ -2,12 +2,33 @@ __author__ = 'aguilbaud' import pexpect -class ping: + +def get_class_name(): + return "Ping" + + +class Ping: def __init__(self, db, addr_host): self.db = db + self.name = get_class_name() + self.compatible_os = [] + self.block = "" + self.unit = 'bool' self.addr_host = addr_host + def get_name(self): + return self.name + + def get_compatible_os(self): + return self.compatible_os + + def get_block(self): + return self.block + + def get_unit(self): + return self.unit + def check(self): res_check = False try: diff --git a/app/modules/monitoring_modules/unix/__init__.py b/app/modules/monitoring_modules/unix/__init__.py deleted file mode 100644 index 0554872..0000000 --- a/app/modules/monitoring_modules/unix/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -__author__ = 'aguilbaud' -__all__=['updated_packages', 'memory', 'disk'] -info_mod = { - 'updated_packages': {'block': 'software', 'unit': 'bool'}, - 'memory': {'block': 'hardware', 'unit': '%'}, - 'disk': {'block' : 'hardware', 'unit': '%'} -} \ No newline at end of file diff --git a/app/modules/monitoring_modules/unix/updated_packages.py b/app/modules/monitoring_modules/unix/updated_packages.py deleted file mode 100644 index 6e66832..0000000 --- a/app/modules/monitoring_modules/unix/updated_packages.py +++ /dev/null @@ -1,16 +0,0 @@ -__author__ = 'alexis' - -class updated_packages: - def __init__(self, conn, db, mnce): - self.conn = conn - self.db = db - - def check(self): - cmd = "apt-get upgrade -s" - stdout = self.conn.exec_command(cmd) - tab_res = stdout.split(':') - if len(tab_res) == 2: - res_check = False - else: - res_check = True - self.db.add_check(self.conn.get_addr_host(), "updated_packages", res_check) \ No newline at end of file diff --git a/app/modules/monitoring_modules/updated_packages.py b/app/modules/monitoring_modules/updated_packages.py new file mode 100644 index 0000000..4359b5c --- /dev/null +++ b/app/modules/monitoring_modules/updated_packages.py @@ -0,0 +1,37 @@ +__author__ = 'aguilbaud' + + +def get_class_name(): + return "UpdatedPackages" + + +class UpdatedPackages: + def __init__(self, conn, db, mnce): + self.conn = conn + self.db = db + self.name = get_class_name() + self.compatible_os = ['linux', 'unix'] + self.block = "software" + self.unit = "bool" + + def get_name(self): + return self.name + + def get_compatible_os(self): + return self.compatible_os + + def get_block(self): + return self.block + + def get_unit(self): + return self.unit + + def check(self): + cmd = "apt-get upgrade -s" + stdout = self.conn.exec_command(cmd) + tab_res = stdout.split(':') + if len(tab_res) == 2: + res_check = False + else: + res_check = True + self.db.add_check(self.conn.get_addr_host(), "updated_packages", res_check) \ No newline at end of file diff --git a/app/modules/storage_modules/shelve_db.py b/app/modules/storage_modules/shelve_db.py index 023f073..e8adf8e 100644 --- a/app/modules/storage_modules/shelve_db.py +++ b/app/modules/storage_modules/shelve_db.py @@ -86,17 +86,24 @@ class shelve_db: By default, only the ping monitoring is activated, while the connection is not configured and tje os not detected :param dict_mod_info: dictionary containing informations about all notification modules, in the form: - [monitoring_module_name][{'block':val, 'unit': val}] + { + mod_name: + { + 'class_name': val, => the name of the class to instanciate + 'compatible_os': [val1, val2, ...], => a list containing the compatibles os + 'unit': val, => the unit type of return ('%', 'bool' or other) + 'block': val, => the monitoring block of the module + 'external': bool => indicates if this modules comes from external directory + } + } :return a list containing the default parameters for each monitoring module (at least ping) """ res = {} if not self.db["hosts"][addr_host]["conf"]["connections"] == {} and not os_host == 'unknown': - if os_host not in dict_mod_info: - print "OS " + os_host + " is not supported by the monitoring modules currently loaded." - else: - for mod in dict_mod_info[os_host]: + for mod in dict_mod_info: + if os_host in dict_mod_info[mod]['compatible_os']: mod_conf = {} - mod_conf['block'] = dict_mod_info[os_host][mod]['block'] + mod_conf['block'] = dict_mod_info[mod]['block'] mod_conf['activated'] = True mod_conf['check_frequency'] = 60 mod_conf['nb_minute'] = 30 @@ -105,7 +112,7 @@ class shelve_db: mod_conf['nb_week'] = 2 mod_conf['nb_month'] = 6 mod_conf['nb_year'] = None - unit = dict_mod_info[os_host][mod]['unit'] + unit = dict_mod_info[mod]['unit'] mod_conf['unit'] = unit if unit == '%': mod_conf['minor_limit'] = 95 diff --git a/app/process_monitoring.py b/app/process_monitoring.py index 32b0806..a3aaee9 100644 --- a/app/process_monitoring.py +++ b/app/process_monitoring.py @@ -90,11 +90,10 @@ class RunMonitoring(threading.Thread): if dict_mod['mod_name'] == 'ping': print "Launching " + str(dict_mod['mod_name']) + " request on " + str(dict_mod['addr']) sys.stdout.flush() - self.ml.run_one_monitoring_module(dict_mod['mod_name'], dict_mod['addr'], None, None, None, None) + self.ml.run_one_monitoring_module(dict_mod['mod_name'], dict_mod['addr'], None, None) else: print "Launching " + str(dict_mod['os']) + "." + \ str(dict_mod['mod_name']) + " on " + str(dict_mod['addr']) sys.stdout.flush() - self.ml.run_one_monitoring_module(dict_mod['mod_name'], - dict_mod['addr'], dict_mod['os'], None, None, None) + self.ml.run_one_monitoring_module(dict_mod['mod_name'], dict_mod['addr'], None, None) self.wsc.notify_state_change() \ No newline at end of file diff --git a/conf.txt b/conf.txt index 544b1a0..008cd5f 100644 --- a/conf.txt +++ b/conf.txt @@ -1,2 +1,2 @@ db_location=mum.db -external_modules_location=~/external/ \ No newline at end of file +external_modules_location=/home/aguilbaud/external \ No newline at end of file -- To stop receiving notification emails like this one, please contact chorem.org SCM administrator <admin+scm@chorem.org>.