Merge pull request #3228 from anarkiwi/gaugerestart

Gauge can detect and reload when FAUCET's config changes.

faucet/config_parser.py

@@ -243,34 +243,44 @@ def _config_parser_v2(config_file, logname, meta_dp_state):
 def watcher_parser(config_file, logname, prom_client):
     """Return Watcher instances from config."""
     conf = config_parser_util.read_config(config_file, logname)
-    return _watcher_parser_v2(conf, logname, prom_client)
+    conf_hash = config_parser_util.config_file_hash(config_file)
+    faucet_config_files, faucet_conf_hashes, result = _watcher_parser_v2(
+        conf, logname, prom_client)
+    return conf_hash, faucet_config_files, faucet_conf_hashes, result
 
 
 def _parse_dps_for_watchers(conf, logname, meta_dp_state=None):
-    dps = {}
-    if 'faucet_configs' in conf:
-        for faucet_file in conf['faucet_configs']:
-            _, dp_list, _ = dp_parser(faucet_file, logname)
-            if dp_list:
-                for dp in dp_list:
-                    dps[dp.name] = dp
-
-    if 'faucet' in conf:
-        faucet_conf = conf['faucet']
-        dps = {dp.name: dp for dp in dp_preparsed_parser(faucet_conf, meta_dp_state)}
+    all_dps_list = []
+    faucet_conf_hashes = {}
 
+    if not isinstance(conf, dict):
+        raise InvalidConfigError('Gauge config not valid')
+
+    faucet_config_files = conf.get('faucet_configs', [])
+    for faucet_config_file in faucet_config_files:
+        conf_hashes, dp_list, _ = dp_parser(faucet_config_file, logname)
+        if dp_list:
+            faucet_conf_hashes[faucet_config_file] = conf_hashes
+            all_dps_list.extend(dp_list)
+
+    faucet_config = conf.get('faucet', None)
+    if faucet_config:
+        all_dps_list.extend(dp_preparsed_parser(faucet_config, meta_dp_state))
+
+    dps = {dp.name: dp for dp in all_dps_list}
     if not dps:
         raise InvalidConfigError(
             'Gauge configured without any FAUCET configuration')
-    return dps
+    return faucet_config_files, faucet_conf_hashes, dps
 
 
 def _watcher_parser_v2(conf, logname, prom_client):
     logger = config_parser_util.get_logger(logname)
 
     if conf is None:
         conf = {}
-    dps = _parse_dps_for_watchers(conf, logname)
+    faucet_config_files, faucet_conf_hashes, dps = _parse_dps_for_watchers(
+        conf, logname)
     dbs = conf.pop('dbs')
 
     result = []
@@ -300,7 +310,7 @@ def _watcher_parser_v2(conf, logname, prom_client):
                 watcher.add_dp(dp)
                 result.append(watcher)
 
-    return result
+    return faucet_config_files, faucet_conf_hashes, result
 
 
 def get_config_for_api(valves):

faucet/gauge.py

@@ -50,6 +50,7 @@ def __init__(self, *args, **kwargs):
         super(Gauge, self).__init__(*args, **kwargs)
         self.watchers = {}
         self.config_watcher = ConfigWatcher()
+        self.faucet_config_watchers = []
         self.prom_client = GaugePrometheusClient(reg=self._reg)
         self.thread_managers = (self.prom_client,)
 
@@ -70,8 +71,10 @@ def _get_watchers(self, ryu_event):
     def _load_config(self):
         """Load Gauge config."""
         try:
-            new_confs = watcher_parser(self.config_file, self.logname, self.prom_client)
+            conf_hash, faucet_config_files, faucet_conf_hashes, new_confs = watcher_parser(
+                self.config_file, self.logname, self.prom_client)
         except InvalidConfigError as err:
+            self.config_watcher.update(self.config_file)
             self.logger.error('invalid config: %s', err)
             return
 
@@ -97,7 +100,14 @@ def _load_config(self):
                 self._start_watchers(ryu_dp, watchers, timestamp)
 
         self.watchers = new_watchers
-        self.config_watcher.update(self.config_file)
+        self.config_watcher.update(
+            self.config_file, {self.config_file: conf_hash})
+        self.faucet_config_watchers = []
+        for faucet_config_file, faucet_conf_hash in faucet_conf_hashes.items():
+            faucet_config_watcher = ConfigWatcher()
+            faucet_config_watcher.update(faucet_config_file, faucet_conf_hash)
+            self.faucet_config_watchers.append(faucet_config_watcher)
+            self.logger.info('watching FAUCET config %s' % faucet_config_file)
         self.logger.info('config complete')
 
     @kill_on_exception(exc_logname)
@@ -111,7 +121,10 @@ def _update_watcher(self, name, ryu_event):
                 watcher.update(ryu_event.timestamp, ryu_dp.id, msg)
 
     def _config_files_changed(self):
-        return self.config_watcher.files_changed()
+        for config_watcher in [self.config_watcher] + self.faucet_config_watchers:
+            if config_watcher.files_changed():
+                return True
+        return False
 
     @set_ev_cls(EventReconfigure, MAIN_DISPATCHER)
     def reload_config(self, ryu_event):

tests/unit/gauge/test_config_gauge.py

@@ -94,7 +94,7 @@ def test_all_dps(self):
 """
         conf = self.get_config(GAUGE_CONF)
         gauge_file, _ = self.create_config_files(conf)
-        watcher_confs = cp.watcher_parser(gauge_file, 'gauge_config_test', None)
+        _, _, _, watcher_confs = cp.watcher_parser(gauge_file, 'gauge_config_test', None)
         self.assertEqual(len(watcher_confs), 2, 'failed to create config for each dp')
         for watcher_conf in watcher_confs:
             msg = 'all_dps config not applied to each dp'
@@ -162,8 +162,9 @@ def test_no_faucet_config_file(self):
         type: 'prometheus'
 """
         gauge_file, _ = self.create_config_files(GAUGE_CONF, '')
-        watcher_conf = cp.watcher_parser(
-            gauge_file, 'gauge_config_test', None)[0]
+        _, _, _, watcher_confs = cp.watcher_parser(
+            gauge_file, 'gauge_config_test', None)
+        watcher_conf = watcher_confs[0]
         msg = 'failed to create watcher correctly when dps configured in gauge.yaml'
         self.assertEqual(watcher_conf.dps[0], 'dp1', msg)
         self.assertEqual(watcher_conf.type, 'port_stats', msg)

tests/unit/gauge/test_gauge.py

@@ -26,7 +26,7 @@
 
 from prometheus_client import CollectorRegistry
 
-from faucet import gauge, gauge_prom, gauge_influx, gauge_pollers, watcher
+from faucet import gauge, gauge_prom, gauge_influx, gauge_pollers, watcher, valve_util
 
 
 class QuietHandler(BaseHTTPRequestHandler):
@@ -853,8 +853,15 @@ def test_flow_stats(self):
 class RyuAppSmokeTest(unittest.TestCase): # pytype: disable=module-attr
 
     def setUp(self):
-        os.environ['GAUGE_LOG'] = '/dev/null'
-        os.environ['GAUGE_EXCEPTION_LOG'] = '/dev/null'
+        self.tmpdir = tempfile.mkdtemp()
+        os.environ['GAUGE_LOG'] = os.path.join(self.tmpdir, 'gauge.log')
+        os.environ['GAUGE_EXCEPTION_LOG'] = os.path.join(self.tmpdir, 'gauge-exception.log')
+        self.ryu_app = None
+
+    def tearDown(self):
+        valve_util.close_logger(self.ryu_app.logger)
+        valve_util.close_logger(self.ryu_app.exc_logger)
+        shutil.rmtree(self.tmpdir)
 
     @staticmethod
     def _fake_dp():
@@ -868,29 +875,28 @@ def _fake_event(self):
         event.dp = msg.datapath
         return event
 
+    def _write_config(self, config_file_name, config):
+        with open(config_file_name, 'w') as config_file:
+            config_file.write(config)
+
     def test_gauge(self):
         """Test Gauge can be initialized."""
         os.environ['GAUGE_CONFIG'] = '/dev/null'
-        ryu_app = gauge.Gauge(
+        self.ryu_app = gauge.Gauge(
             dpset={},
             reg=CollectorRegistry())
-        ryu_app.reload_config(None)
-        self.assertFalse(ryu_app._config_files_changed())
-        ryu_app._update_watcher(None, self._fake_event())
-        ryu_app._start_watchers(self._fake_dp(), {}, time.time())
+        self.ryu_app.reload_config(None)
+        self.assertFalse(self.ryu_app._config_files_changed())
+        self.ryu_app._update_watcher(None, self._fake_event())
+        self.ryu_app._start_watchers(self._fake_dp(), {}, time.time())
         for event_handler in (
-                ryu_app._datapath_connect,
-                ryu_app._datapath_disconnect):
+                self.ryu_app._datapath_connect,
+                self.ryu_app._datapath_disconnect):
             event_handler(self._fake_event())
 
     def test_gauge_config(self):
         """Test Gauge minimal config."""
-        tmpdir = tempfile.mkdtemp()
-        os.environ['FAUCET_CONFIG'] = os.path.join(tmpdir, 'faucet.yaml')
-        os.environ['GAUGE_CONFIG'] = os.path.join(tmpdir, 'gauge.yaml')
-        with open(os.environ['FAUCET_CONFIG'], 'w') as faucet_config:
-            faucet_config.write(
-                """
+        faucet_conf1 = """
 vlans:
    100:
        description: "100"
@@ -901,11 +907,23 @@ def test_gauge_config(self):
            1:
                description: "1"
                native_vlan: 100
-""")
-        os.environ['GAUGE_CONFIG'] = os.path.join(tmpdir, 'gauge.yaml')
-        with open(os.environ['GAUGE_CONFIG'], 'w') as gauge_config:
-            gauge_config.write(
-                """
+"""
+        faucet_conf2 = """
+vlans:
+   100:
+       description: "200"
+dps:
+   dp1:
+       dp_id: 0x1
+       interfaces:
+           2:
+               description: "2"
+               native_vlan: 100
+"""
+        os.environ['FAUCET_CONFIG'] = os.path.join(self.tmpdir, 'faucet.yaml')
+        self._write_config(os.environ['FAUCET_CONFIG'], faucet_conf1)
+        os.environ['GAUGE_CONFIG'] = os.path.join(self.tmpdir, 'gauge.yaml')
+        gauge_conf = """
 faucet_configs:
    - '%s'
 watchers:
@@ -928,15 +946,36 @@ def test_gauge_config(self):
         type: 'prometheus'
         prometheus_addr: '0.0.0.0'
         prometheus_port: 0
-""" % os.environ['FAUCET_CONFIG'])
-        ryu_app = gauge.Gauge(
+""" % os.environ['FAUCET_CONFIG']
+        self._write_config(os.environ['GAUGE_CONFIG'], gauge_conf)
+        self.ryu_app = gauge.Gauge(
             dpset={},
             reg=CollectorRegistry())
-        ryu_app.reload_config(None)
-        self.assertTrue(ryu_app.watchers)
-        ryu_app.reload_config(None)
-        self.assertTrue(ryu_app.watchers)
-        shutil.rmtree(tmpdir)
+        self.ryu_app.reload_config(None)
+        self.assertFalse(self.ryu_app._config_files_changed())
+        self.assertTrue(self.ryu_app.watchers)
+        self.ryu_app.reload_config(None)
+        self.assertTrue(self.ryu_app.watchers)
+        self.assertFalse(self.ryu_app._config_files_changed())
+        # Load a new FAUCET config.
+        self._write_config(os.environ['FAUCET_CONFIG'], faucet_conf2)
+        self.assertTrue(self.ryu_app._config_files_changed())
+        self.ryu_app.reload_config(None)
+        self.assertTrue(self.ryu_app.watchers)
+        self.assertFalse(self.ryu_app._config_files_changed())
+        # Load an invalid Gauge config
+        self._write_config(os.environ['GAUGE_CONFIG'], 'invalid')
+        self.assertTrue(self.ryu_app._config_files_changed())
+        self.ryu_app.reload_config(None)
+        self.assertTrue(self.ryu_app.watchers)
+        # Keep trying to load a valid version.
+        self.assertTrue(self.ryu_app._config_files_changed())
+        # Load good Gauge config back
+        self._write_config(os.environ['GAUGE_CONFIG'], gauge_conf)
+        self.assertTrue(self.ryu_app._config_files_changed())
+        self.ryu_app.reload_config(None)
+        self.assertTrue(self.ryu_app.watchers)
+        self.assertFalse(self.ryu_app._config_files_changed())
 
 
 if __name__ == "__main__":