diff --git a/openwisp_controller/subnet_division/rule_types/base.py b/openwisp_controller/subnet_division/rule_types/base.py index 826150cbb..ef2bfb61f 100644 --- a/openwisp_controller/subnet_division/rule_types/base.py +++ b/openwisp_controller/subnet_division/rule_types/base.py @@ -1,79 +1,53 @@ import logging from ipaddress import ip_network from operator import attrgetter - from django.core.exceptions import ObjectDoesNotExist -from django.db import connection, transaction +from django.db import transaction from django.dispatch import Signal from django.utils.translation import gettext_lazy as _ -from netaddr import IPNetwork +from netaddr import IPNetwork, IPSet from openwisp_notifications.signals import notify from swapper import load_model - from ..signals import subnet_provisioned logger = logging.getLogger(__name__) - Subnet = load_model("openwisp_ipam", "Subnet") IpAddress = load_model("openwisp_ipam", "IpAddress") SubnetDivisionRule = load_model("subnet_division", "SubnetDivisionRule") SubnetDivisionIndex = load_model("subnet_division", "SubnetDivisionIndex") -VpnClient = load_model("config", "VpnClient") class BaseSubnetDivisionRuleType(object): - provision_signal = None - provision_sender = None - provision_dispatch_uid = None - - destroyer_signal = None - destroyer_sender = None - destroyer_dispatch_uid = None - - organization_id_path = None - subnet_path = None + provision_signal = provision_sender = provision_dispatch_uid = None + destroyer_signal = destroyer_sender = destroyer_dispatch_uid = None + organization_id_path = subnet_path = None config_path = "config" @classmethod def validate_rule_type(cls): assert issubclass(cls, BaseSubnetDivisionRuleType) - assert isinstance(cls.provision_signal, Signal) - assert isinstance(cls.provision_dispatch_uid, str) cls.provision_sender = load_model(*cls.provision_sender) - - assert isinstance(cls.destroyer_signal, Signal) - assert isinstance(cls.destroyer_dispatch_uid, str) cls.destroyer_sender = load_model(*cls.destroyer_sender) - assert isinstance(cls.organization_id_path, str) - assert isinstance(cls.subnet_path, str) - @classmethod def provision_receiver(cls, instance, **kwargs): def _provision_receiver(): - # If any of following operations fail, the database transaction - # should fail/rollback. - - # This method is also called by "provision_for_existing_objects" - # which passes the "rule" keyword argument. In such case, - # provisioning should be only triggered for received rule. - if "rule" in kwargs: - rules = [kwargs["rule"]] - else: - try: - rules = cls.get_subnet_division_rules(instance) - except (AttributeError, ObjectDoesNotExist): - return + try: + rules = ( + [kwargs["rule"]] + if "rule" in kwargs + else cls.get_subnet_division_rules(instance) + ) + except (AttributeError, ObjectDoesNotExist): + return for rule in rules: provisioned = cls.create_subnets_ips(instance, rule, **kwargs) cls.post_provision_handler(instance, provisioned, **kwargs) cls.subnet_provisioned_signal_emitter(instance, provisioned) - if not cls.should_create_subnets_ips(instance, **kwargs): - return - - transaction.on_commit(_provision_receiver) + if cls.should_create_subnets_ips(instance, **kwargs): + transaction.on_commit(_provision_receiver) @classmethod def destroyer_receiver(cls, instance, **kwargs): @@ -81,13 +55,6 @@ def destroyer_receiver(cls, instance, **kwargs): @staticmethod def post_provision_handler(instance, provisioned, **kwargs): - """ - This method should be overridden in inherited rule types to - perform any operation on provisioned subnets and IP addresses. - :param instance: object that triggered provisioning - :param provisioned: dictionary containing subnets and IP addresses - provisioned, None if nothing is provisioned - """ pass @staticmethod @@ -98,38 +65,12 @@ def subnet_provisioned_signal_emitter(instance, provisioned): @classmethod def should_create_subnets_ips(cls, instance, **kwargs): - """ - return a boolean value whether subnets and IPs should - be provisioned for "instance" object - """ raise NotImplementedError() @classmethod def provision_for_existing_objects(cls, rule_obj): - """ - Contains logic to trigger provisioning for existing objects - """ raise NotImplementedError() - @classmethod - def create_subnets_ips(cls, instance, division_rule, **kwargs): - try: - config = cls.get_config(instance) - except (AttributeError, ObjectDoesNotExist): - return - - master_subnet = division_rule.master_subnet - max_subnet = cls.get_max_subnet(master_subnet, division_rule) - generated_indexes = [] - generated_subnets = cls.create_subnets( - config, division_rule, max_subnet, generated_indexes - ) - generated_ips = cls.create_ips( - config, division_rule, generated_subnets, generated_indexes - ) - SubnetDivisionIndex.objects.bulk_create(generated_indexes) - return {"subnets": generated_subnets, "ip_addresses": generated_ips} - @classmethod def get_organization(cls, instance): return attrgetter(cls.organization_id_path)(instance) @@ -138,158 +79,155 @@ def get_organization(cls, instance): def get_subnet(cls, instance): return attrgetter(cls.subnet_path)(instance) + @classmethod + def get_config(cls, instance): + p = cls.config_path or "config" + conf = instance if p == "self" else attrgetter(p)(instance) + if not conf._meta.model.objects.filter(pk=conf.pk).exists(): + raise ObjectDoesNotExist() + return conf + @classmethod def get_subnet_division_rules(cls, instance): - rule_type = f"{cls.__module__}.{cls.__name__}" - organization_id = cls.get_organization(instance) - subnet = cls.get_subnet(instance) + rt = f"{cls.__module__}.{cls.__name__}" + org_id, subnet = cls.get_organization(instance), cls.get_subnet(instance) return subnet.subnetdivisionrule_set.filter( - organization_id__in=(organization_id, None), - type=rule_type, + organization_id__in=(org_id, None), type=rt ).iterator() - @classmethod - def get_config(cls, instance): - if cls.config_path == "self": - config = instance - else: - config = attrgetter(cls.config_path)(instance) - # check for real existence in DB to workaround - # this django-import-export bug: - # https://github.com/django-import-export/django-import-export/issues/1078 - # TODO: if that issue is ever solved, we can remove the this block below - Config = config._meta.model - if not Config.objects.filter(pk=config.pk).exists(): - raise ObjectDoesNotExist() - return config - @staticmethod - def get_max_subnet(master_subnet, division_rule): - # Only PostgreSQL supports ordering queryset using the "subnet" - # field. If the project is using any other database backend, then - # "created" field is used for ordering the queryset. - order_field = "-subnet" if connection.vendor == "postgresql" else "-created" - try: - max_subnet = ( - # Get the highest subnet created for this master_subnet - Subnet.objects.filter(master_subnet_id=master_subnet.id) - .order_by(order_field) - .first() - .subnet - ) - except AttributeError: - # If there is no existing subnet, create a reserved subnet - # and use it as starting point - required_subnet = next( - IPNetwork(str(master_subnet.subnet)).subnet( - prefixlen=division_rule.size - ) - ) - subnet_obj = Subnet( - name=f"Reserved Subnet {required_subnet}", - subnet=str(required_subnet), - description=_("Automatically generated reserved subnet."), - master_subnet_id=master_subnet.id, - organization_id=master_subnet.organization_id, + def get_max_subnet(ms, rule): + qs = Subnet.objects.filter(master_subnet_id=ms.id) + rule_nets = list( + qs.filter(subnetdivisionindex__rule=rule).values_list("subnet", flat=True) + ) + if not rule_nets: + m_net = IPNetwork(str(ms.subnet)) + anchor = next(m_net.subnet(prefixlen=rule.size)) + obj, created = Subnet.objects.get_or_create( + subnet=str(anchor), + master_subnet_id=ms.id, + defaults={ + "name": f"Reserved Subnet {anchor}", + "description": _("Anchor point to protect address."), + "organization_id": ms.organization_id, + }, ) - subnet_obj.full_clean() - subnet_obj.save() - max_subnet = subnet_obj.subnet - return max_subnet + return str(obj.subnet) + parsed = [ip_network(str(s)) for s in rule_nets] + return str(max(parsed, key=lambda n: int(n.network_address))) @staticmethod - def create_subnets(config, division_rule, max_subnet, generated_indexes): - master_subnet = division_rule.master_subnet - required_subnet = IPNetwork(str(max_subnet)).next() - generated_subnets = [] - - for subnet_id in range(1, division_rule.number_of_subnets + 1): - if not ip_network(str(required_subnet)).subnet_of(master_subnet.subnet): + def create_subnets(config, rule, max_s, idxs): + ms = rule.master_subnet + exist = ( + Subnet.objects.filter( + master_subnet_id=ms.id, + subnetdivisionindex__rule=rule, + subnetdivisionindex__config=config, + ) + .values("id") + .distinct() + .count() + ) + if exist >= rule.number_of_subnets: + return [] + req = IPNetwork(str(max_s)) + a_qs = Subnet.objects.filter(subnet=max_s, master_subnet=ms) + if exist > 0 or (a_qs.exists() and "Reserved" in (a_qs.first().name or "")): + req = req.next() + subs, space = [], IPSet( + [str(s.subnet) for s in Subnet.objects.filter(master_subnet_id=ms.id)] + ) + for s_id in range(exist + 1, rule.number_of_subnets + 1): + while ip_network(str(req)).subnet_of(ms.subnet) and space.intersection( + IPSet([req]) + ): + req = req.next() + if not ip_network(str(req)).subnet_of(ms.subnet): notify.send( sender=config, type="generic_message", target=config.device, - action_object=master_subnet, level="error", + action_object=ms, message=_( - "Failed to provision subnets for" - " [{notification.target}]({notification.target_link})" + "Failed to provision subnets for " + "[{notification.target}]({notification.target_link})" ), description=_( - "The [{notification.action_object}]({notification.action_link})" - " subnet has run out of space." + "The [{notification.action_object}]" + "({notification.action_link}) subnet has run " + "out of space." ), ) - logger.info(f"Cannot create more subnets of {master_subnet}") + logger.info(f"Cannot create more subnets of {ms.name} {ms.subnet}") break - subnet_obj = Subnet( - name=f"{division_rule.label}_subnet{subnet_id}", - subnet=str(required_subnet), - description=_( - f"Automatically generated using {division_rule.label} rule." - ), - master_subnet_id=master_subnet.id, - organization_id=division_rule.organization_id, + obj = Subnet.objects.create( + name=f"{rule.label}_subnet{s_id}", + subnet=str(req), + description=_("Automatically provisioned subnet."), + master_subnet_id=ms.id, + organization_id=rule.organization_id, ) - subnet_obj.full_clean() - generated_subnets.append(subnet_obj) - generated_indexes.append( + subs.append(obj) + space.update(IPSet([req])) + idxs.append( SubnetDivisionIndex( - keyword=f"{division_rule.label}_subnet{subnet_id}", - subnet_id=subnet_obj.id, - rule_id=division_rule.id, + keyword=f"{rule.label}_subnet{s_id}", + subnet_id=obj.id, + rule_id=rule.id, config=config, ) ) - required_subnet = required_subnet.next() - Subnet.objects.bulk_create(generated_subnets) - return generated_subnets + return subs @staticmethod - def create_ips(config, division_rule, generated_subnets, generated_indexes): - generated_ips = [] - for subnet_obj in generated_subnets: - # don't assign first ip address of a subnet, - # unless the rule is designed to use the whole - # address space of the subnet - if subnet_obj.subnet.num_addresses != division_rule.number_of_ips: - index_start = 1 - index_end = division_rule.number_of_ips + 1 - # this allows handling /32, /128 or cases in which - # the number of requested ip addresses matches exactly - # what is available in the subnet - else: - index_start = 0 - index_end = division_rule.number_of_ips - # generate IPs and indexes accordingly - for ip_index in range(index_start, index_end): - ip_obj = IpAddress( - subnet_id=subnet_obj.id, - ip_address=str(subnet_obj.subnet[ip_index]), - ) - ip_obj.full_clean() - generated_ips.append(ip_obj) - # ensure human friendly labels (starting from 1 instead of 0) - keyword_index = ip_index if index_start == 1 else ip_index + 1 - generated_indexes.append( + def create_ips(config, rule, subnets, idxs): + all_ips = [] + for s_obj in subnets: + net = IPNetwork(str(s_obj.subnet)) + is_full = rule.number_of_ips >= len(net) + start = 0 if (net.prefixlen in [32, 128] or is_full) else 1 + count = 0 + for ip_index in range(start, len(net)): + if count >= rule.number_of_ips: + break + addr = str(net[ip_index]) + obj = IpAddress.objects.create(subnet_id=s_obj.id, ip_address=addr) + all_ips.append(obj) + count += 1 + idxs.append( SubnetDivisionIndex( - keyword=f"{subnet_obj.name}_ip{keyword_index}", - subnet_id=subnet_obj.id, - ip_id=ip_obj.id, - rule_id=division_rule.id, + keyword=f"{s_obj.name}_ip{count}", + subnet_id=s_obj.id, + ip_id=obj.id, + rule_id=rule.id, config=config, ) ) - IpAddress.objects.bulk_create(generated_ips) - return generated_ips + return all_ips + + @classmethod + def create_subnets_ips(cls, instance, division_rule, **kwargs): + try: + config = cls.get_config(instance) + except (AttributeError, ObjectDoesNotExist): + return + with transaction.atomic(): + ms = division_rule.master_subnet + max_s = cls.get_max_subnet(ms, division_rule) + idxs = [] + subs = cls.create_subnets(config, division_rule, max_s, idxs) + ips = cls.create_ips(config, division_rule, subs, idxs) + SubnetDivisionIndex.objects.bulk_create(idxs) + return {"subnets": subs, "ip_addresses": ips} @classmethod def destroy_provisioned_subnets_ips(cls, instance, **kwargs): - # Deleting related subnets automatically deletes related IpAddress - # and SubnetDivisionIndex objects config = cls.get_config(instance) - rule_type = f"{cls.__module__}.{cls.__name__}" - subnet_ids = config.subnetdivisionindex_set.filter( - rule__type=rule_type - ).values_list("subnet_id") - Subnet.objects.filter(id__in=subnet_ids).delete() + rt = f"{cls.__module__}.{cls.__name__}" + s_ids = config.subnetdivisionindex_set.filter(rule__type=rt).values_list( + "subnet_id", flat=True + ) + Subnet.objects.filter(id__in=s_ids).delete()