Static Routes with nexthop non-functional for private gateways (#12859)

* Fix static routes to be added to PBR tables in VPC routers

Static routes were only being added to the main routing table, but
policy-based routing (PBR) is active on VPC routers. This caused
traffic coming in from specific interfaces to not find the static
routes, as they use interface-specific routing tables (Table_ethX).

This fix:
- Adds a helper method to find which interface a gateway belongs to
  by matching the gateway IP against configured interface subnets
- Modifies route add/delete operations to update both the main table
  and the appropriate interface-specific PBR table
- Uses existing CsAddress databag metadata to avoid OS queries
- Handles both add and revoke operations for proper cleanup
- Adds comprehensive logging for troubleshooting

Fixes #12857

* Add iptables FORWARD rules for nexthop-based static routes

When static routes use nexthop (gateway) instead of referencing a
private gateway's public IP, the iptables FORWARD rules were not
being generated. This caused traffic to be dropped by ACLs.

This fix:
- Adds a shared helper CsHelper.find_device_for_gateway() to determine
  which interface a gateway belongs to by checking subnet membership
- Updates CsStaticRoutes to use the shared helper instead of duplicating
  the device-finding logic
- Modifies CsAddress firewall rule generation to handle both old-style
  (ip_address-based) and new-style (nexthop-based) static routes
- Generates the required FORWARD and PREROUTING rules for nexthop routes:
  * -A PREROUTING -s <network> ! -d <interface_ip>/32 -i <dev> -j ACL_OUTBOUND_<dev>
  * -A FORWARD -d <network> -o <dev> -j ACL_INBOUND_<dev>
  * -A FORWARD -d <network> -o <dev> -m state --state RELATED,ESTABLISHED -j ACCEPT

Fixes the second part of #12857

* network matching grep fix, don't let 1.2.3.4/32 match 11.2.3.4/32

Author: bhouse-nexthop

systemvm/debian/opt/cloud/bin/cs/CsAddress.py

@@ -584,6 +584,37 @@ def fw_vpcrouter(self):
                                 "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" %
                                 (self.dev, guestNetworkCidr, self.address['gateway'], self.dev)])
 
+        # Process static routes for this interface
+        static_routes = CsStaticRoutes("staticroutes", self.config)
+        if static_routes:
+            for item in static_routes.get_bag():
+                if item == "id":
+                    continue
+                static_route = static_routes.get_bag()[item]
+                if static_route['revoke']:
+                    continue
+
+                # Check if this static route applies to this interface
+                # Old style: ip_address field matches this interface's public_ip
+                # New style (nexthop): gateway is in this interface's subnet
+                applies_to_interface = False
+                if 'ip_address' in static_route and static_route['ip_address'] == self.address['public_ip']:
+                    applies_to_interface = True
+                elif 'gateway' in static_route:
+                    device = CsHelper.find_device_for_gateway(self.config, static_route['gateway'])
+                    if device == self.dev:
+                        applies_to_interface = True
+
+                if applies_to_interface:
+                    self.fw.append(["mangle", "",
+                                    "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" %
+                                    (self.dev, static_route['network'], self.address['public_ip'], self.dev)])
+                    self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %
+                                    (static_route['network'], self.dev, self.dev)])
+                    self.fw.append(["filter", "front",
+                                    "-A FORWARD -d %s -o %s -m state --state RELATED,ESTABLISHED -j ACCEPT" %
+                                    (static_route['network'], self.dev)])
+
         if self.is_private_gateway():
             self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %
                             (self.address['network'], self.dev, self.dev)])
@@ -597,22 +628,6 @@ def fw_vpcrouter(self):
                             "-A PREROUTING -s %s -d %s -m state --state NEW -j MARK --set-xmark %s/0xffffffff" %
                             (self.cl.get_vpccidr(), self.address['network'], hex(100 + int(self.dev[3:])))])
 
-            static_routes = CsStaticRoutes("staticroutes", self.config)
-            if static_routes:
-                for item in static_routes.get_bag():
-                    if item == "id":
-                        continue
-                    static_route = static_routes.get_bag()[item]
-                    if 'ip_address' in static_route and static_route['ip_address'] == self.address['public_ip'] and not static_route['revoke']:
-                        self.fw.append(["mangle", "",
-                                        "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" %
-                                        (self.dev, static_route['network'], static_route['ip_address'], self.dev)])
-                        self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" %
-                                        (static_route['network'], self.dev, self.dev)])
-                        self.fw.append(["filter", "front",
-                                        "-A FORWARD -d %s -o %s -m state --state RELATED,ESTABLISHED -j ACCEPT" %
-                                        (static_route['network'], self.dev)])
-
             if self.address["source_nat"]:
                 self.fw.append(["nat", "front",
                                 "-A POSTROUTING -o %s -j SNAT --to-source %s" %

systemvm/debian/opt/cloud/bin/cs/CsHelper.py

@@ -25,8 +25,12 @@
 import os.path
 import re
 import shutil
+from typing import Optional, TYPE_CHECKING
 from netaddr import *
 
+if TYPE_CHECKING:
+    from .CsConfig import CsConfig
+
 PUBLIC_INTERFACES = {"router": "eth2", "vpcrouter": "eth1"}
 
 STATE_COMMANDS = {"router": "ip addr show dev eth0 | grep inet | wc -l | xargs bash -c  'if [ $0 == 2 ]; then echo \"PRIMARY\"; else echo \"BACKUP\"; fi'",
@@ -270,3 +274,29 @@ def copy(src, dest):
         logging.error("Could not copy %s to %s" % (src, dest))
     else:
         logging.info("Copied %s to %s" % (src, dest))
+
+
+def find_device_for_gateway(config: 'CsConfig', gateway_ip: str) -> Optional[str]:
+    """
+    Find which ethernet device the gateway IP belongs to by checking
+    if the gateway is in any of the configured interface subnets.
+
+    Args:
+        config: CsConfig instance containing network configuration
+        gateway_ip: IP address of the gateway to locate
+
+    Returns:
+        Device name (e.g., 'eth2') or None if not found
+    """
+    try:
+        interfaces = config.address().get_interfaces()
+        for interface in interfaces:
+            if not interface.is_added():
+                continue
+            if interface.ip_in_subnet(gateway_ip):
+                return interface.get_device()
+        logging.debug("No matching device found for gateway %s" % gateway_ip)
+        return None
+    except Exception as e:
+        logging.error("Error finding device for gateway %s: %s" % (gateway_ip, e))
+        return None

systemvm/debian/opt/cloud/bin/cs/CsStaticRoutes.py

@@ -20,6 +20,7 @@
 import logging
 from . import CsHelper
 from .CsDatabag import CsDataBag
+from .CsRoute import CsRoute
 
 
 class CsStaticRoutes(CsDataBag):
@@ -31,13 +32,46 @@ def process(self):
                 continue
             self.__update(self.dbag[item])
 
+
+
     def __update(self, route):
+        network = route['network']
+        gateway = route['gateway']
+
         if route['revoke']:
-            command = "ip route del %s via %s" % (route['network'], route['gateway'])
+            # Delete from main table
+            command = "ip route del %s via %s" % (network, gateway)
             CsHelper.execute(command)
+
+            # Delete from PBR table if applicable
+            device = CsHelper.find_device_for_gateway(self.config, gateway)
+            if device:
+                cs_route = CsRoute()
+                table_name = cs_route.get_tablename(device)
+                command = "ip route del %s via %s table %s" % (network, gateway, table_name)
+                CsHelper.execute(command)
+                logging.info("Deleted static route %s via %s from PBR table %s" % (network, gateway, table_name))
         else:
-            command = "ip route show | grep %s | awk '{print $1, $3}'" % route['network']
+            # Add to main table (existing logic)
+            command = "ip route show | grep '^%s' | awk '{print $1, $3}'" % network
             result = CsHelper.execute(command)
             if not result:
-                route_command = "ip route add %s via %s" % (route['network'], route['gateway'])
+                route_command = "ip route add %s via %s" % (network, gateway)
                 CsHelper.execute(route_command)
+                logging.info("Added static route %s via %s to main table" % (network, gateway))
+
+            # Add to PBR table if applicable
+            device = CsHelper.find_device_for_gateway(self.config, gateway)
+            if device:
+                cs_route = CsRoute()
+                table_name = cs_route.get_tablename(device)
+                # Check if route already exists in the PBR table
+                check_command = "ip route show table %s | grep '^%s' | awk '{print $1, $3}'" % (table_name, network)
+                result = CsHelper.execute(check_command)
+                if not result:
+                    # Add route to the interface-specific table
+                    route_command = "ip route add %s via %s dev %s table %s" % (network, gateway, device, table_name)
+                    CsHelper.execute(route_command)
+                    logging.info("Added static route %s via %s to PBR table %s" % (network, gateway, table_name))
+            else:
+                logging.info("Static route %s via %s added to main table only (no matching interface found for PBR table)" % (network, gateway))