Merge pull request #131 from DanSheps/nb-34-update

Update for NetBox 3.4

Author: DanSheps

.github/workflows/pypi.yml

@@ -28,7 +28,7 @@ jobs:
         name: python-package-distributions
         path: dist/
   publish-to-testpypi:
-    name: Publish Python 🐍 distribution 📦 to PyPI
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
     needs:
       - build
     runs-on: ubuntu-latest
@@ -46,8 +46,8 @@ jobs:
     - name: Publish package to TestPyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        repository_url: https://test.pypi.org/legacy/
-        skip_existing: true
+        repository-url: https://test.pypi.org/legacy/
+        skip-existing: true
   publish-to-pypi:
     name: Publish Python 🐍 distribution 📦 to PyPI
     needs:
@@ -67,4 +67,4 @@ jobs:
     - name: Publish package
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        skip_existing: true
+        skip-existing: true

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
+  hooks:
+    - id: ruff
+      name: "Ruff linter"
+      args: [ netbox_config_backup/ ]
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 25.1.0
+  hooks:
+    - id: black
+      name: "Black"
+

netbox_config_backup/__init__.py

@@ -1,12 +1,6 @@
-from datetime import datetime
 from importlib.metadata import metadata
 
-from django.utils import timezone
-from django_rq import get_queue
-
-from core.choices import JobStatusChoices
 from netbox.plugins import PluginConfig
-from netbox_config_backup.utils.logger import get_logger
 
 metadata = metadata('netbox_config_backup')
 
@@ -19,8 +13,7 @@ class NetboxConfigBackup(PluginConfig):
     author = metadata.get('Author')
     author_email = metadata.get('Author-email')
     base_url = 'configbackup'
-    min_version = '4.1.0'
-    max_version = '4.2.99'
+    min_version = '4.4.0'
     required_settings = [
         'repository',
         'committer',
@@ -30,16 +23,15 @@ class NetboxConfigBackup(PluginConfig):
         # Frequency in seconds
         'frequency': 3600,
     }
-    queues = [
-        'jobs'
-    ]
+    queues = ['jobs']
     graphql_schema = 'graphql.schema.schema'
 
     def ready(self, *args, **kwargs):
         super().ready()
         import sys
+
         if len(sys.argv) > 1 and 'rqworker' in sys.argv[1]:
-            from netbox_config_backup.jobs.backup import BackupRunner
+            from netbox_config_backup.jobs.backup import BackupRunner  # noqa: F401
 
 
 config = NetboxConfigBackup

netbox_config_backup/api/__init__.py

@@ -1 +1 @@
-from .serializers import *
+from .serializers import *

netbox_config_backup/api/views.py

@@ -1,9 +1,8 @@
-
 from netbox.api.viewsets import NetBoxModelViewSet
 from netbox_config_backup.api import BackupSerializer
 from netbox_config_backup.models import Backup
 
 
 class BackupViewSet(NetBoxModelViewSet):
     queryset = Backup.objects.all()
-    serializer_class = BackupSerializer
+    serializer_class = BackupSerializer

netbox_config_backup/backup/processing.py

@@ -1,11 +1,9 @@
 import logging
 import os
-import time
 import traceback
 from datetime import timedelta
 
 import uuid
-from django.db.models import Q
 from django.utils import timezone
 
 from core.choices import JobStatusChoices
@@ -17,32 +15,39 @@
 from netbox_config_backup.utils.napalm import napalm_init
 from netbox_config_backup.utils.rq import can_backup
 
-logger = logging.getLogger(f"netbox_config_backup")
+logger = logging.getLogger("netbox_config_backup")
 
 
 def remove_stale_backupjobs(job: BackupJob):
     pass
 
+
 def run_backup(job_id):
     close_db()
     logger.info(f'Starting backup for job {job_id}')
     try:
+        logger.debug(f'Trying to load job {job_id}')
         job = BackupJob.objects.get(pk=job_id)
     except Exception as e:
         logger.error(f'Unable to load job {job_id}: {e}')
         logger.debug(f'\t{traceback.format_exc()}')
         raise e
 
     try:
+        logger.debug(f'Getting backup for {job}')
         backup = Backup.objects.get(pk=job.backup.pk)
         backup.refresh_from_db()
         pid = os.getpid()
 
+        logger.debug(f'Setting status and saving for {job}')
+
         job.status = JobStatusChoices.STATUS_PENDING
         job.pid = pid
         job.save()
 
+        logger.debug(f'Checking backup status for {job}')
         if not can_backup(backup):
+            logger.info(f'Cannot backup {backup}')
             job.status = JobStatusChoices.STATUS_FAILED
             if not job.data:
                 job.data = {}
@@ -60,20 +65,38 @@ def run_backup(job_id):
             raise e
 
         if ip:
+            logger.debug(
+                f'Trying to connect to device {backup.device} with ip {ip} for {job}'
+            )
             try:
                 d = napalm_init(backup.device, ip)
             except (TimeoutError, ServiceUnavailable):
                 job.status = JobStatusChoices.STATUS_FAILED
-                job.data = {'error': f'Timeout Connecting to {backup.device} with ip {ip}'}
-                logger.debug = f'Timeout Connecting to {backup.device} with ip {ip}'
+                job.data = {
+                    'error': f'Timeout Connecting to {backup.device} with ip {ip}'
+                }
+                logger.debug(f'Timeout Connecting to {backup.device} with ip {ip}')
                 job.save()
                 return
-
+            logger.debug(f'Connected to {backup.device} with ip {ip} for {job}')
             job.status = JobStatusChoices.STATUS_RUNNING
             job.started = timezone.now()
             job.save()
             try:
-                status = check_config_save_status(d)
+                logger.debug(f'Checking config save status for {backup}')
+                config_save_status = check_config_save_status(d)
+                status = config_save_status.get('status')
+                running = backup.files.filter(type='running').first()
+                startup = backup.files.filter(type='startup').first()
+                if running.last_change != config_save_status.get('running', None):
+                    running.last_change = config_save_status.get('running', None)
+                    running.clean()
+                    running.save()
+                if startup.last_change != config_save_status.get('startup', None):
+                    startup.last_change = config_save_status.get('startup', None)
+                    startup.clean()
+                    startup.save()
+
                 if status is not None:
                     if status and not backup.config_status:
                         backup.config_status = status
@@ -88,15 +111,23 @@ def run_backup(job_id):
                         backup.config_status = status
                         backup.save()
             except Exception as e:
+
                 logger.error(f'{backup}: had error setting backup status: {e}')
 
+            logger.debug(f'Getting config for {backup}')
             configs = d.get_config()
+            logger.debug(f'Committing config for {backup}')
             commit = backup.set_config(configs)
-
+            logger.debug(
+                f'Committed config for {backup} with {commit}; closing connection for {backup}'
+            )
             d.close()
 
+            logger.debug(f'Scheduling next backup for {backup}')
             frequency = timedelta(
-                seconds=settings.PLUGINS_CONFIG.get('netbox_config_backup', {}).get('frequency', 3600)
+                seconds=settings.PLUGINS_CONFIG.get('netbox_config_backup', {}).get(
+                    'frequency', 3600
+                )
             )
             new = BackupJob(
                 runner=None,
@@ -115,6 +146,7 @@ def run_backup(job_id):
             job.save()
             remove_stale_backupjobs(job=job)
         else:
+            logger.debug(f'{backup}: No IP set')
             job.status = JobStatusChoices.STATUS_FAILED
             if not job.data:
                 job.data = {}

netbox_config_backup/choices.py

@@ -5,6 +5,7 @@
 # File Types for Backup Files
 #
 
+
 class FileTypeChoices(ChoiceSet):
 
     TYPE_RUNNING = 'running'
@@ -15,6 +16,7 @@ class FileTypeChoices(ChoiceSet):
         (TYPE_STARTUP, 'Startup'),
     )
 
+
 class CommitTreeChangeTypeChoices(ChoiceSet):
 
     TYPE_ADD = 'add'

netbox_config_backup/exceptions/__init__.py

@@ -0,0 +1,2 @@
+class JobExit(Exception):
+    pass

netbox_config_backup/filtersets.py

@@ -5,13 +5,10 @@
 from django.utils.translation import gettext as _
 from netaddr import AddrFormatError
 
-from core.choices import JobStatusChoices
-from ipam.models import IPAddress
-from netbox.filtersets import NetBoxModelFilterSet, BaseFilterSet
+from netbox.filtersets import BaseFilterSet
 from dcim.models import Device
 from netbox_config_backup import models
 from netbox_config_backup.choices import FileTypeChoices
-from utilities.filters import MultiValueCharFilter
 
 
 class BackupJobFilterSet(BaseFilterSet):
@@ -24,6 +21,17 @@ class Meta:
         model = models.BackupJob
         fields = ['id', 'status']
 
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        qs_filter = (
+            Q(backup__name__icontains=value)
+            | Q(backup__ip__address__icontains=value)
+            | Q(backup__device__name__icontains=value)
+        )
+
+        return queryset.filter(qs_filter)
+
 
 class BackupFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
@@ -34,12 +42,16 @@ class BackupFilterSet(BaseFilterSet):
         field_name='device__name',
         queryset=Device.objects.all(),
         to_field_name='name',
-        label='Device (name)',
+        label=_('Device (name)'),
     )
     device_id = django_filters.ModelMultipleChoiceFilter(
         field_name='device',
         queryset=Device.objects.all(),
-        label='Device (ID)',
+        label=_('Device (ID)'),
+    )
+    config_status = django_filters.BooleanFilter(
+        field_name='config_status',
+        label=_('Config Saved'),
     )
     ip = django_filters.CharFilter(
         method='filter_address',
@@ -54,11 +66,11 @@ def search(self, queryset, name, value):
         if not value.strip():
             return queryset
         qs_filter = (
-            Q(name__icontains=value) |
-            Q(device__name__icontains=value) |
-            Q(device__primary_ip4__address__contains=value.strip()) |
-            Q(device__primary_ip6__address__contains=value.strip()) |
-            Q(ip__address__contains=value.strip())
+            Q(name__icontains=value)
+            | Q(device__name__icontains=value)
+            | Q(device__primary_ip4__address__contains=value.strip())
+            | Q(device__primary_ip6__address__contains=value.strip())
+            | Q(ip__address__contains=value.strip())
         )
 
         try:
@@ -81,7 +93,7 @@ def filter_address(self, queryset, name, value):
                 return queryset.filter(query)
             else:
                 return queryset.filter(ip__address__net_host_contained=value)
-        except ValidationError as e:
+        except ValidationError:
             return queryset.none()
 
 
@@ -91,9 +103,7 @@ class BackupsFilterSet(BaseFilterSet):
         label=_('Search'),
     )
     type = django_filters.MultipleChoiceFilter(
-        field_name='file__type',
-        choices=FileTypeChoices,
-        null_value=None
+        field_name='file__type', choices=FileTypeChoices, null_value=None
     )
 
     class Meta:
@@ -103,8 +113,5 @@ class Meta:
     def search(self, queryset, name, value):
         if not value.strip():
             return queryset
-        qs_filter = (
-            Q(file__type=value) |
-            Q(file__type__startswith=value)
-        )
+        qs_filter = Q(file__type=value) | Q(file__type__startswith=value)
         return queryset.filter(qs_filter)