Added automatic model reloading

Author: belavina

docs/AssetsConfigurations.md

@@ -1,14 +1,13 @@
 
 # Asset Configurations
 
-## UPS
-
-### Agents 
+## Agents 
 
 You can check if ipmi/snmp simulators are up & running by issuing status command:
 
 `simengine-cli status --asset-key={key} --agent`
 
+## UPS
 
 ### SNMP Configurations
 

docs/Installation.md

@@ -20,6 +20,7 @@ Simengine uses OpenIPMI lanserv simulator for its BMC emulations and libvirt for
 ```
 dnf install libvirt OpenIPMI OpenIPMI-lanserv OpenIPMI-libs OpenIPMI-devel python3-libvirt -y
 dnf install gcc redis -y
+dnf install ipmitool -y #for testing
 ```
 
 ## Neo4J
@@ -62,6 +63,29 @@ Change location to `dashboard/fronend`
 
 Run `npm install` and then `npm start`
 
+## MIBs
+
+Vendor-specific mibs may need to be installed for the testing purposes.
+
+`dnf install net-snmp net-snmp-utils`
+
+Add mib definitions:
+
+`mkdir /usr/share/snmp/mibs/apc`
+
+`cp data/ups/powernet426.mib /usr/share/snmp/mibs/apc/ # copy from simengine project`
+
+Create configuration file:
+
+`vi /etc/snmp/snmp.conf`
+
+Paste `mibdirs` reference:
+
+```
+mibdirs /usr/share/snmp/mibs:/usr/share/snmp/mibs/apc
+mibs ALL
+```
+
 ## Running
 
 `simengine-cli` will need to be put into `$PATH` as `export PATH="$PATH:/path/to/simengine/enginecore"`

enginecore/app.py

@@ -7,8 +7,6 @@
 import tempfile
 
 from enginecore.state.state_listener import StateListener
-from enginecore.state.state_initializer import initialize
-
 
 def configure_env(relative=False):
     """Set-up defaults for the env vars if not defined 
@@ -33,20 +31,6 @@ def configure_env(relative=False):
     )
 
 
-def clear_temp():
-    """All app data is stored in /tmp/simengine (which is cleared on restart)"""
-    sys_temp = tempfile.gettempdir()
-    simengine_temp = os.path.join(sys_temp, 'simengine')
-    if os.path.exists(simengine_temp):
-        for the_file in os.listdir(simengine_temp):
-            file_path = os.path.join(simengine_temp, the_file)    
-            if os.path.isfile(file_path):
-                os.unlink(file_path)
-            elif os.path.isdir(file_path): 
-                shutil.rmtree(file_path)
-    else:
-        os.makedirs(simengine_temp)
-
 def run():
     """
     Initilize compnents' states in redis based on a reference model
@@ -64,17 +48,11 @@ def run():
 
     args = vars(argparser.parse_args())
 
-    # set-up temp app space
-    clear_temp()
-
     # env space configuration
     configure_env(relative=args['develop'])
 
-    # init state
-    initialize(force_snmp_init=args['reload_data'])
-
     # run daemon
-    StateListener(debug=args['verbose']).run()
+    StateListener(debug=args['verbose'], force_snmp_init=args['reload_data']).run()
 
 if __name__ == '__main__':
     run()

enginecore/enginecore/state/redis_channels.py

@@ -10,6 +10,9 @@ class RedisChannels():
     load_update_channel = 'load-upd'
     state_update_channel = 'state-upd'
     oid_update_channel = 'oid-upd'
+    model_update_channel = 'model-upd'
+
+    # battery states
     battery_update_channel = 'battery-upd'
     battery_conf_drain_channel = 'battery-drain-upd'
     battery_conf_charge_channel = 'battery-charge-upd'

enginecore/enginecore/state/state_initializer.py

@@ -1,9 +1,29 @@
 """Initialize redis state based on reference model """
 import os
+import tempfile
+import shutil
+
 import redis
+
 from enginecore.model.graph_reference import GraphReference
 from enginecore.state.utils import format_as_redis_key, get_asset_type
 
+
+def clear_temp():
+    """All app data is stored in /tmp/simengine (which is cleared on restart)"""
+    sys_temp = tempfile.gettempdir()
+    simengine_temp = os.path.join(sys_temp, 'simengine')
+    if os.path.exists(simengine_temp):
+        for the_file in os.listdir(simengine_temp):
+            file_path = os.path.join(simengine_temp, the_file)    
+            if os.path.isfile(file_path):
+                os.unlink(file_path)
+            elif os.path.isdir(file_path): 
+                shutil.rmtree(file_path)
+    else:
+        os.makedirs(simengine_temp)
+
+
 def initialize(force_snmp_init=False):
     """ Initialize redis state using topology defined in the graph db """
 

enginecore/enginecore/state/state_listener.py

@@ -19,6 +19,7 @@
 from enginecore.state.web_socket import WebSocket
 from enginecore.state.redis_channels import RedisChannels
 from enginecore.model.graph_reference import GraphReference
+from enginecore.state.state_initializer import initialize, clear_temp
 
 class NotifyClient(Event):
     """Notify websocket clients of any data updates"""
@@ -27,36 +28,26 @@ class StateListener(Component):
     """Top-level component that instantiates assets & maps redis events to circuit events"""
 
 
-    def __init__(self, debug=False):
+    def __init__(self, debug=False, force_snmp_init=False):
         super(StateListener, self).__init__()
 
         ### Set-up WebSocket & Redis listener ###
 
-        # subscribe to redis key events
-        self.redis_store = redis.StrictRedis(host='localhost', port=6379)
+        # Use redis pub/sub communication
+        self._redis_store = redis.StrictRedis(host='localhost', port=6379)
 
-        # State Channels
-        self.pubsub = self.redis_store.pubsub()
-        self.pubsub.psubscribe(
-            RedisChannels.oid_update_channel,  # snmp oid updates
-            RedisChannels.state_update_channel # power state changes
-        )
-
-        # Battery Channel
-        self.bat_pubsub = self.redis_store.pubsub()
-        self.bat_pubsub.psubscribe(
-            RedisChannels.battery_update_channel, # battery level updates
-            RedisChannels.battery_conf_drain_channel, # update drain speed (factor)
-            RedisChannels.battery_conf_charge_channel # update charge speed (factor)
-        )
+        self._bat_pubsub = self._redis_store.pubsub()
+        self._state_pubsub = self._redis_store.pubsub()
 
         # assets will store all the devices/items including PDUs, switches etc.
         self._assets = {}
+
+        # init graph db instance
         self._graph_ref = GraphReference()
-        self._graph_db = self._graph_ref.get_session()
 
-        # set up a web socket
-        self._server = Server(("0.0.0.0", 8000)).register(self)     
+        # set up a web socket server
+        self._server = Server(("0.0.0.0", 8000)).register(self)  
+
         # Worker(process=False).register(self)
         Static().register(self._server)
         Logger().register(self._server)
@@ -65,10 +56,39 @@ def __init__(self, debug=False):
             Debugger(events=False).register(self)
 
         self._ws = WebSocket().register(self._server)
-    
         WebSocketsDispatcher("/simengine").register(self._server)
 
         ### Register Assets ###
+        self._subscribe_to_channels()
+        self._reload_model(force_snmp_init)
+       
+
+    def _subscribe_to_channels(self):
+        """Subscribe to redis channels"""
+        
+        # State Channels
+        self._state_pubsub.psubscribe(
+            RedisChannels.oid_update_channel,  # snmp oid updates
+            RedisChannels.state_update_channel, # power state changes
+            RedisChannels.model_update_channel  # model changes
+        )
+
+        # Battery Channel
+        self._bat_pubsub.psubscribe(
+            RedisChannels.battery_update_channel, # battery level updates
+            RedisChannels.battery_conf_drain_channel, # update drain speed (factor)
+            RedisChannels.battery_conf_charge_channel # update charge speed (factor)
+        )
+
+
+    def _reload_model(self, force_snmp_init=True):
+        """Re-create system topology (instantiate assets based on graph ref)"""
+
+        self._assets = {}
+
+        # init state
+        clear_temp()
+        initialize(force_snmp_init) 
 
         # instantiate assets based on graph records
         leaf_nodes = []
@@ -147,10 +167,6 @@ def _handle_state_update(self, asset_key):
         self._chain_power_update(PowerEventResult(asset_key=asset_key, new_state=asset_status))
 
 
-    def __exit__(self, exc_type, exc_value, traceback):
-        self._graph_db.close()
-
-
     def _chain_load_update(self, event_result, increased=True):
         """React to load update event by propogating the load changes up the power stream
         
@@ -317,7 +333,7 @@ def _notify_client(self, asset_key, data):
 
     def monitor_battery(self):
         """Monitor battery in a separate pub/sub stream"""
-        message = self.bat_pubsub.get_message()
+        message = self._bat_pubsub.get_message()
 
         # validate message
         if ((not message) or ('data' not in message) or (not isinstance(message['data'], bytes))):
@@ -346,7 +362,7 @@ def monitor_state(self):
         """ listens to redis events """
 
         print("...")
-        message = self.pubsub.get_message()
+        message = self._state_pubsub.get_message()
 
         # validate message
         if ((not message) or ('data' not in message) or (not isinstance(message['data'], bytes))):
@@ -364,14 +380,22 @@ def monitor_state(self):
                     self._handle_state_update(int(asset_key))
 
             elif message['channel'] == str.encode(RedisChannels.oid_update_channel):
-                value = (self.redis_store.get(data)).decode()
+                value = (self._redis_store.get(data)).decode()
                 asset_key, oid = data.split('-')
                 self._handle_oid_update(int(asset_key), oid, value)
 
             elif message['channel'] == str.encode(RedisChannels.battery_update_channel):
                 asset_key, _ = data.split('-')
                 self._notify_client(int(asset_key), {'battery': self._assets[int(asset_key)].state.battery_level})
 
+            elif message['channel'] == str.encode(RedisChannels.model_update_channel):
+                print('RELOAD REQUESTED')
+                self._state_pubsub.unsubscribe()
+                self._bat_pubsub.unsubscribe()
+
+                self._reload_model()
+                self._subscribe_to_channels()
+
         except KeyError as error:
             print("Detected unregistered asset under key [{}]".format(error), file=sys.stderr)
 

enginecore/enginecore/state/state_managers.py

@@ -295,8 +295,16 @@ def get_system_status(cls, flatten=True):
             return assets
 
 
+    @classmethod
+    def reload_model(cls):
+        """Request daemon reloading"""
+        StateManager.get_store().publish(RedisChannels.model_update_channel, 'reload')
+
+
     @classmethod
     def get_state_manager_by_key(cls, key, supported_assets, notify=True):
+        """Infer asset manager from key"""
+
         graph_ref = GraphReference()
 
         with graph_ref.get_session() as session:  

enginecore/simengine-cli

@@ -7,10 +7,8 @@ import json
 import time
 import curses
 from enginecore.state.assets import Asset
-from enginecore.model.graph_reference import GraphReference
 import enginecore.model.system_modeler as sm
 from enginecore.state.state_managers import StateManager
-from enginecore.state.utils import get_asset_type
 
 
 def manage_state(asset_key, mng_action):
@@ -48,10 +46,16 @@ def status_table_format(assets, stdscr=False):
     for i, asset_key in enumerate(assets):
         asset = assets[asset_key]
         children = str(asset['children'] if 'children' in asset else "none")
-        row = row_format.format(str(i), *[str(asset_key), asset['type'], str(asset['status']), children, "{0:.2f}".format(asset['load'])], end='')
+        row = row_format.format(
+            str(i), 
+            *[str(asset_key), asset['type'], str(asset['status']), children, "{0:.2f}".format(asset['load'])], 
+            end=''
+        )
 
         if stdscr:
-            stdscr.addstr(i+1, 0, row, curses.color_pair(bcolors.ERROR if int(asset['status']) == 0 else bcolors.OKGREEN))
+            stdscr.addstr(
+                i+1, 0, row, curses.color_pair(bcolors.ERROR if int(asset['status']) == 0 else bcolors.OKGREEN)
+            )
         else:
             print(row)
 
@@ -132,10 +136,10 @@ def get_status(**kwargs):
 
 def configure_battery(key, kwargs):
     """Udpate runtime battery status"""
-    if kwargs['drain_speed'] != None:
+    if kwargs['drain_speed'] is not None:
         state_manager = Asset.get_state_manager_by_key(key, notify=True)
         state_manager.set_drain_speed_factor(kwargs['drain_speed'])
-    if kwargs['charge_speed'] != None:
+    if kwargs['charge_speed'] is not None:
         state_manager = Asset.get_state_manager_by_key(key, notify=True)
         state_manager.set_charge_speed_factor(kwargs['charge_speed'])
 
@@ -203,6 +207,8 @@ asset_group = subparsers.add_parser('model', help="Manage system model: create n
 model_subp = asset_group.add_subparsers()
 create_asset_group = model_subp.add_parser('create', help="Create new asset")
 
+reload_asset_action = model_subp.add_parser('reload', help="Reload the system topology (notify daemon of model changes)")
+
 # detach & delete an asset by key
 delete_asset_action = model_subp.add_parser('delete', help="Remove individual asset by key")
 delete_asset_action.add_argument('-k', '--asset-key', type=int, required=True)
@@ -572,6 +578,10 @@ create_static_action.set_defaults(
 for action in update_actions:
     action.set_defaults(func=lambda args: sm.configure_asset(args['asset_key'], args))
 
+reload_asset_action.set_defaults(
+    func=lambda args: StateManager.reload_model() 
+)
+
 delete_asset_action.set_defaults(
     func=lambda args: sm.delete_asset(args['asset_key'])
 )

enginecore/tests/server_load_m1.py

@@ -33,8 +33,6 @@ class ServerLoadTest(unittest.TestCase):
     def setUpClass(cls):
         cls.redis_store = redis.StrictRedis(host='localhost', port=6379)
 
-    def setUp(self):
-
         server_attr = {
             'domain_name': 'an-a01n01', 
             'psu_num': 2, 
@@ -64,6 +62,9 @@ def setUp(self):
         sm.link_assets(33, 5)
         sm.link_assets(2, 42)
 
+        StateManager.reload_model()
+        time.sleep(3)
+
 
     def check_redis_values(self, expected_kv):
         for key, value in expected_kv.items():