statd: optimize yanger invocation time and reduce command overhead

troglobit · troglobit · commit 263f2fe82a40 · 2026-03-02T06:26:01.000+01:00
Replace logging + logging.handlers with a lightweight syslog wrapper,
and argparse with manual argv parsing.  On a sama7g54, this cuts yanger
startup from ~770ms to ~470ms by eliminating ~300ms of stdlib imports.

Also batch external command invocations:

 - ietf_routing: two sysctl calls instead of two per interface
 - ietf_hardware: one ls per hwmon device instead of six
 - bridge: fetch mctl querier data once instead of once per VLAN

Signed-off-by: Joachim Wiberg &lt;troglobit@gmail.com&gt;
diff --git a/src/statd/python/yanger/__main__.py b/src/statd/python/yanger/__main__.py
@@ -1,97 +1,130 @@
-import logging
-import logging.handlers
 import json
-import sys  # (built-in module)
 import os
-import argparse
+import sys
 
 from . import common
 from . import host
 
-def main():
-    def dirpath(path):
-        if not os.path.isdir(path):
-            raise argparse.ArgumentTypeError(f"'{path}' is not a valid directory")
-        return path
+USAGE = """\
+usage: yanger [-p PARAM] [-x PREFIX] [-r DIR | -c DIR] model
 
-    parser = argparse.ArgumentParser(description="YANG data creator")
-    parser.add_argument("model", help="YANG Model")
-    parser.add_argument("-p", "--param",
-                        help="Model dependent parameter, e.g. interface name")
-    parser.add_argument("-x", "--cmd-prefix", metavar="PREFIX",
-                        help="Use this prefix for all system commands, e.g. " +
-                        "'ssh user@remotehost sudo'")
+YANG data creator
 
-    rrparser = parser.add_mutually_exclusive_group()
-    rrparser.add_argument("-r", "--replay", type=dirpath, metavar="DIR",
-                          help="Generate output based on recorded system commands from DIR, " +
-                          "rather than querying the local system")
-    rrparser.add_argument("-c", "--capture", metavar="DIR",
-                          help="Capture system command output in DIR, such that the current system " +
-                          "state can be recreated offline (with --replay) for testing purposes")
+positional arguments:
+  model                 YANG Model
 
-    args = parser.parse_args()
-    if args.replay and args.cmd_prefix:
-        parser.error("--cmd-prefix cannot be used with --replay")
+options:
+  -p, --param PARAM     Model dependent parameter, e.g. interface name
+  -x, --cmd-prefix PREFIX
+                        Use this prefix for all system commands, e.g.
+                        'ssh user@remotehost sudo'
+  -r, --replay DIR      Generate output based on recorded system commands
+                        from DIR, rather than querying the local system
+  -c, --capture DIR     Capture system command output in DIR, such that the
+                        current system state can be recreated offline (with
+                        --replay) for testing purposes
+"""
 
-    # Set up syslog output for critical errors to aid debugging
-    common.LOG = logging.getLogger('yanger')
-    if os.path.exists('/dev/log'):
-        log = logging.handlers.SysLogHandler(address='/dev/log')
-    else:
-        # Use /dev/null as a fallback for unit tests
-        log = logging.FileHandler('/dev/null')
+def _parse_args(argv):
+    model = None
+    param = None
+    cmd_prefix = None
+    replay = None
+    capture = None
+
+    i = 1
+    while i < len(argv):
+        arg = argv[i]
+        if arg in ('-h', '--help'):
+            sys.stdout.write(USAGE)
+            sys.exit(0)
+        elif arg in ('-p', '--param'):
+            i += 1
+            if i >= len(argv):
+                sys.exit(f"error: {arg} requires an argument")
+            param = argv[i]
+        elif arg in ('-x', '--cmd-prefix'):
+            i += 1
+            if i >= len(argv):
+                sys.exit(f"error: {arg} requires an argument")
+            cmd_prefix = argv[i]
+        elif arg in ('-r', '--replay'):
+            i += 1
+            if i >= len(argv):
+                sys.exit(f"error: {arg} requires an argument")
+            replay = argv[i]
+            if not os.path.isdir(replay):
+                sys.exit(f"error: '{replay}' is not a valid directory")
+        elif arg in ('-c', '--capture'):
+            i += 1
+            if i >= len(argv):
+                sys.exit(f"error: {arg} requires an argument")
+            capture = argv[i]
+        elif arg.startswith('-'):
+            sys.exit(f"error: unknown option: {arg}")
+        elif model is None:
+            model = arg
+        else:
+            sys.exit(f"error: unexpected argument: {arg}")
+        i += 1
 
-    fmt = logging.Formatter('%(name)s[%(process)d]: %(message)s')
-    log.setFormatter(fmt)
-    common.LOG.setLevel(logging.INFO)
-    common.LOG.addHandler(log)
+    if model is None:
+        sys.exit("error: missing required argument: model")
+    if replay and cmd_prefix:
+        sys.exit("error: --cmd-prefix cannot be used with --replay")
+    if replay and capture:
+        sys.exit("error: --replay cannot be used with --capture")
+
+    return model, param, cmd_prefix, replay, capture
+
+def main():
+    model, param, cmd_prefix, replay, capture = _parse_args(sys.argv)
 
-    if args.cmd_prefix or args.capture:
-        host.HOST = host.Remotehost(args.cmd_prefix, args.capture)
-    elif args.replay:
-        host.HOST = host.Replayhost(args.replay)
+    if cmd_prefix or capture:
+        host.HOST = host.Remotehost(cmd_prefix, capture)
+    elif replay:
+        host.HOST = host.Replayhost(replay)
     else:
         host.HOST = host.Localhost()
 
-    if args.model == 'ietf-interfaces':
+    if model == 'ietf-interfaces':
         from . import ietf_interfaces
-        yang_data = ietf_interfaces.operational(args.param)
-    elif args.model == 'ietf-routing':
+        yang_data = ietf_interfaces.operational(param)
+    elif model == 'ietf-routing':
         from . import ietf_routing
         yang_data = ietf_routing.operational()
-    elif args.model == 'ietf-ospf':
+    elif model == 'ietf-ospf':
         from . import ietf_ospf
         yang_data = ietf_ospf.operational()
-    elif args.model == 'ietf-rip':
+    elif model == 'ietf-rip':
         from . import ietf_rip
         yang_data = ietf_rip.operational()
-    elif args.model == 'ietf-hardware':
+    elif model == 'ietf-hardware':
         from . import ietf_hardware
         yang_data = ietf_hardware.operational()
-    elif args.model == 'infix-containers':
+    elif model == 'infix-containers':
         from . import infix_containers
         yang_data = infix_containers.operational()
-    elif args.model == 'infix-dhcp-server':
+    elif model == 'infix-dhcp-server':
         from . import infix_dhcp_server
         yang_data = infix_dhcp_server.operational()
-    elif args.model == 'ietf-system':
+    elif model == 'ietf-system':
         from . import ietf_system
         yang_data = ietf_system.operational()
-    elif args.model == 'ietf-ntp':
+    elif model == 'ietf-ntp':
         from . import ietf_ntp
         yang_data = ietf_ntp.operational()
-    elif args.model == 'ieee802-dot1ab-lldp':
+    elif model == 'ieee802-dot1ab-lldp':
         from . import infix_lldp
         yang_data = infix_lldp.operational()
-    elif args.model == 'infix-firewall':
+    elif model == 'infix-firewall':
         from . import infix_firewall
         yang_data = infix_firewall.operational()
-    elif args.model == 'ietf-bfd-ip-sh':
+    elif model == 'ietf-bfd-ip-sh':
         from . import ietf_bfd_ip_sh
         yang_data = ietf_bfd_ip_sh.operational()
     else:
-        common.LOG.warning("Unsupported model %s", args.model)
+        common.LOG.warning("Unsupported model %s", model)
         sys.exit(1)
 
     print(json.dumps(yang_data, indent=2, ensure_ascii=False))
diff --git a/src/statd/python/yanger/common.py b/src/statd/python/yanger/common.py
@@ -1,8 +1,50 @@
+import syslog
 from datetime import timedelta
 
 from . import host
 
-LOG = None
+
+class SysLog:
+    """Lightweight syslog wrapper replacing the logging module.
+
+    Provides the same .error()/.warning()/.info()/.debug() interface
+    used throughout yanger, but uses the C syslog facility directly,
+    avoiding the ~374ms import overhead of logging + logging.handlers.
+    """
+
+    DEBUG = syslog.LOG_DEBUG
+    INFO = syslog.LOG_INFO
+    WARNING = syslog.LOG_WARNING
+    ERROR = syslog.LOG_ERR
+
+    def __init__(self, name):
+        syslog.openlog(name, syslog.LOG_PID)
+        self._level = self.INFO
+
+    def setLevel(self, level):
+        self._level = level
+
+    def _log(self, level, msg, *args):
+        if level > self._level:
+            return
+        if args:
+            msg = msg % args
+        syslog.syslog(level, msg)
+
+    def debug(self, msg, *args):
+        self._log(self.DEBUG, msg, *args)
+
+    def info(self, msg, *args):
+        self._log(self.INFO, msg, *args)
+
+    def warning(self, msg, *args):
+        self._log(self.WARNING, msg, *args)
+
+    def error(self, msg, *args):
+        self._log(self.ERROR, msg, *args)
+
+
+LOG = SysLog("yanger")
 
 class YangDate:
     def __init__(self, dt=None):
diff --git a/src/statd/python/yanger/ietf_hardware.py b/src/statd/python/yanger/ietf_hardware.py
@@ -265,8 +265,11 @@ def create_sensor(sensor_name, value, value_type, value_scale, label=None):
                         component["description"] = desc
                     return component
 
+                # List hwmon directory once, reuse for all sensor types
+                all_entries = HOST.run(("ls", hwmon_path), default="").split()
+
                 # Temperature sensors
-                temp_entries = HOST.run(("ls", hwmon_path), default="").split()
+                temp_entries = all_entries
                 temp_files = [os.path.join(hwmon_path, e) for e in temp_entries if e.startswith("temp") and e.endswith("_input")]
                 for temp_file in temp_files:
                     try:
@@ -285,7 +288,7 @@ def create_sensor(sensor_name, value, value_type, value_scale, label=None):
                         continue
 
                 # Fan sensors (RPM from tachometer)
-                fan_entries = HOST.run(("ls", hwmon_path), default="").split()
+                fan_entries = all_entries
                 fan_files = [os.path.join(hwmon_path, e) for e in fan_entries if e.startswith("fan") and e.endswith("_input")]
                 for fan_file in fan_files:
                     try:
@@ -307,7 +310,7 @@ def create_sensor(sensor_name, value, value_type, value_scale, label=None):
                 # Only add if no fan*_input exists for this device (avoid duplicates)
                 has_rpm_sensor = bool(fan_files)
                 if not has_rpm_sensor:
-                    pwm_entries = HOST.run(("ls", hwmon_path), default="").split()
+                    pwm_entries = all_entries
                     pwm_files = [os.path.join(hwmon_path, e) for e in pwm_entries if e.startswith("pwm") and e[3:].replace('_', '').isdigit() if len(e) > 3]
                     for pwm_file in pwm_files:
                         # Skip pwm*_enable, pwm*_mode, etc. - only process pwm1, pwm2, etc.
@@ -336,7 +339,7 @@ def create_sensor(sensor_name, value, value_type, value_scale, label=None):
                             continue
 
                 # Voltage sensors
-                voltage_entries = HOST.run(("ls", hwmon_path), default="").split()
+                voltage_entries = all_entries
                 voltage_files = [os.path.join(hwmon_path, e) for e in voltage_entries if e.startswith("in") and e.endswith("_input")]
                 for voltage_file in voltage_files:
                     try:
@@ -356,7 +359,7 @@ def create_sensor(sensor_name, value, value_type, value_scale, label=None):
                         continue
 
                 # Current sensors
-                current_entries = HOST.run(("ls", hwmon_path), default="").split()
+                current_entries = all_entries
                 current_files = [os.path.join(hwmon_path, e) for e in current_entries if e.startswith("curr") and e.endswith("_input")]
                 for current_file in current_files:
                     try:
@@ -376,7 +379,7 @@ def create_sensor(sensor_name, value, value_type, value_scale, label=None):
                         continue
 
                 # Power sensors
-                power_entries = HOST.run(("ls", hwmon_path), default="").split()
+                power_entries = all_entries
                 power_files = [os.path.join(hwmon_path, e) for e in power_entries if e.startswith("power") and e.endswith("_input")]
                 for power_file in power_files:
                     try:
diff --git a/src/statd/python/yanger/ietf_interfaces/bridge.py b/src/statd/python/yanger/ietf_interfaces/bridge.py
@@ -204,10 +204,13 @@ def mctlq2yang_mode(mctlq):
     return "off"
 
 
-def mctl(ifname, vid):
-    mctl = HOST.run_json(["mctl", "-p", "show", "igmp", "json"], default={})
+def mctl_queriers():
+    """Fetch all IGMP multicast querier data in one call"""
+    return HOST.run_json(["mctl", "-p", "show", "igmp", "json"], default={})
 
-    for q in mctl.get("multicast-queriers", []):
+
+def mctl(ifname, vid, mctldata):
+    for q in mctldata.get("multicast-queriers", []):
         # TODO: Also need to match against VLAN uppers (e.g. br0.1337)
         if q.get("interface") == ifname and q.get("vid") == vid:
             return q
@@ -239,8 +242,8 @@ def multicast_filters(iplink, vid):
     return { "multicast-filter": list(mdb.values()) }
 
 
-def multicast(iplink, info):
-    mctlq = mctl(iplink["ifname"], info.get("vlan"))
+def multicast(iplink, info, mctldata):
+    mctlq = mctl(iplink["ifname"], info.get("vlan"), mctldata)
 
     mcast = {
         "snooping": bool(info.get("mcast_snooping")),
@@ -276,13 +279,15 @@ def vlans(iplink):
     if not (brgvlans := HOST.run_json(f"bridge -j vlan global show dev {iplink['ifname']}".split())):
         return []
 
+    mctldata = mctl_queriers()
+
     vlans = {
         v["vlan"]: {
             "vid": v["vlan"],
             "untagged": [],
             "tagged": [],
 
-            "multicast": multicast(iplink, v),
+            "multicast": multicast(iplink, v, mctldata),
             "multicast-filters": multicast_filters(iplink, v["vlan"]),
         }
         for v in brgvlans[0]["vlans"]
@@ -307,7 +312,7 @@ def dbridge(iplink):
     info = iplink["linkinfo"]["info_data"]
 
     return {
-        "multicast": multicast(iplink, info),
+        "multicast": multicast(iplink, info, mctl_queriers()),
         "multicast-filters": multicast_filters(iplink, None),
     }
 
diff --git a/src/statd/python/yanger/ietf_routing.py b/src/statd/python/yanger/ietf_routing.py
@@ -132,19 +132,34 @@ def get_routing_interfaces():
     links_json = HOST.run(tuple(['ip', '-j', 'link', 'show']), default="[]")
     links = json.loads(links_json)
 
+    # Fetch all forwarding sysctls in two calls instead of 2 per interface
+    ipv4_sysctls = HOST.run(tuple(['sysctl', 'net.ipv4.conf']), default="")
+    ipv6_sysctls = HOST.run(tuple(['sysctl', 'net.ipv6.conf']), default="")
+
+    # Parse "net.ipv4.conf.<iface>.forwarding = 1" lines into a set
+    ipv4_fwd = set()
+    ipv6_fwd = set()
+    for line in ipv4_sysctls.splitlines():
+        if '.forwarding = 1' in line:
+            # net.ipv4.conf.IFNAME.forwarding = 1
+            parts = line.split('.')
+            if len(parts) >= 5:
+                ipv4_fwd.add(parts[3])
+
+    for line in ipv6_sysctls.splitlines():
+        if '.force_forwarding = 1' in line:
+            # net.ipv6.conf.IFNAME.force_forwarding = 1
+            parts = line.split('.')
+            if len(parts) >= 5:
+                ipv6_fwd.add(parts[3])
+
     routing_ifaces = []
     for link in links:
         ifname = link.get('ifname')
         if not ifname:
             continue
 
-        # Check if IPv4 forwarding is enabled
-        ipv4_fwd = HOST.run(tuple(['sysctl', '-n', f'net.ipv4.conf.{ifname}.forwarding']), default="0").strip()
-
-        # Check if IPv6 force_forwarding is enabled (available since Linux 6.17)
-        ipv6_fwd = HOST.run(tuple(['sysctl', '-n', f'net.ipv6.conf.{ifname}.force_forwarding']), default="0").strip()
-
-        if ipv4_fwd == "1" or ipv6_fwd == "1":
+        if ifname in ipv4_fwd or ifname in ipv6_fwd:
             routing_ifaces.append(ifname)
 
     return routing_ifaces
