Prepare v38.1.0 for release

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Release notes
 =============
 
+Version v38.1.0
+---------------------
+
+- Throttle UI to 15 requests per minute to avoid abuse and improve performance.
+- Handle errors in unfurl_version_range pipeline.
+- Remove Todo pipeline from v1 pipelines.
+- Add openAPI documentation for Package and Advisory viewset.
+
 Version v38.0.0
 ---------------------
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 38.0.0
+version = 38.1.0
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

vulnerabilities/api_v3.py

@@ -14,6 +14,7 @@
 from django.db.models import OuterRef
 from django.db.models import Prefetch
 from django_filters import rest_framework as filters
+from drf_spectacular.utils import extend_schema
 from packageurl import PackageURL
 from rest_framework import serializers
 from rest_framework import viewsets
@@ -422,6 +423,7 @@ class PackageV3ViewSet(viewsets.GenericViewSet):
     filter_backends = [filters.DjangoFilterBackend]
     throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle]
 
+    @extend_schema(request=PackageQuerySerializer)
     def create(self, request, *args, **kwargs):
         serializer = PackageQuerySerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
@@ -528,8 +530,9 @@ class AdvisoryV3ViewSet(viewsets.GenericViewSet):
     filter_backends = [filters.DjangoFilterBackend]
     throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle]
 
+    @extend_schema(request=AdvisoryQuerySerializer)
     def create(self, request, *args, **kwargs):
-        serializer = PackageQuerySerializer(data=request.data)
+        serializer = AdvisoryQuerySerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
 
         purls = serializer.validated_data["purls"]

vulnerabilities/importers/__init__.py

@@ -21,7 +21,6 @@
 from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
-from vulnerabilities.importers import openssl
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
 from vulnerabilities.importers import project_kb_msr2019
@@ -38,7 +37,6 @@
 from vulnerabilities.pipelines import gitlab_importer
 from vulnerabilities.pipelines import nginx_importer
 from vulnerabilities.pipelines import npm_importer
-from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
 from vulnerabilities.pipelines import pysec_importer
 from vulnerabilities.pipelines.v2_importers import alpine_linux_importer as alpine_linux_importer_v2
@@ -118,7 +116,6 @@
         retiredotnet_importer_v2.RetireDotnetImporterPipeline,
         ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline,
         alpine_linux_importer_v2.AlpineLinuxImporterPipeline,
-        nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,
         gitlab_importer.GitLabImporterPipeline,
         github_osv.GithubOSVImporter,
@@ -136,7 +133,6 @@
         alpine_linux_importer.AlpineLinuxImporterPipeline,
         ruby.RubyImporter,
         apache_kafka.ApacheKafkaImporter,
-        openssl.OpensslImporter,
         openssl_importer_v2.OpenSSLImporterPipeline,
         redhat.RedhatImporter,
         archlinux.ArchlinuxImporter,

vulnerabilities/improvers/__init__.py

@@ -10,7 +10,6 @@
 from vulnerabilities.improvers import valid_versions
 from vulnerabilities.improvers import vulnerability_status
 from vulnerabilities.pipelines import add_cvss31_to_CVEs
-from vulnerabilities.pipelines import compute_advisory_todo
 from vulnerabilities.pipelines import compute_package_risk
 from vulnerabilities.pipelines import compute_package_version_rank
 from vulnerabilities.pipelines import enhance_with_exploitdb
@@ -70,7 +69,6 @@
         compute_package_risk_v2.ComputePackageRiskPipeline,
         compute_version_rank_v2.ComputeVersionRankPipeline,
         unfurl_version_range_v2.UnfurlVersionRangePipeline,
-        compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
         group_advisories_for_packages.GroupAdvisoriesForPackages,

vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py

@@ -52,7 +52,7 @@ def unfurl_version_range(self):
             if purl.type not in RANGE_CLASS_BY_SCHEMES:
                 continue
 
-            versions = get_purl_versions(purl, cached_versions)
+            versions = get_purl_versions(purl, cached_versions) or []
             affected_purls = get_affected_purls(
                 versions=versions,
                 affecting_vers=impact.affecting_vers,
@@ -79,6 +79,8 @@ def get_affected_purls(versions, affecting_vers, base_purl, logger):
     version_class = affecting_version_range.version_class
 
     try:
+        if not versions:
+            return []
         versions = [version_class(v) for v in versions]
     except Exception as e:
         logger(
@@ -107,8 +109,10 @@ def get_affected_purls(versions, affecting_vers, base_purl, logger):
 
 def get_purl_versions(purl, cached_versions):
     if not purl in cached_versions:
-        cached_versions[purl] = get_versions(purl)
-    return cached_versions[purl]
+        purls = get_versions(purl)
+        if purls is not None:
+            cached_versions[purl] = purls
+    return cached_versions.get(purl) or []
 
 
 def bulk_create_with_m2m(purls, impact, relation, logger):

vulnerabilities/tests/test_view.py

@@ -11,8 +11,10 @@
 import time
 
 import pytest
+from django.core.cache import cache
 from django.test import Client
 from django.test import TestCase
+from django.urls import reverse
 from packageurl import PackageURL
 from univers import versions
 
@@ -330,3 +332,25 @@ def test_aggregate_fixed_and_affected_packages(self):
             end_time = time.time()
             assert end_time - start_time < 0.05
             self.assertEqual(response.status_code, 200)
+
+
+class ThrottleTestCase(TestCase):
+    def setUp(self):
+        self.client = Client()
+        cache.clear()
+
+    def test_throttle_after_15_requests(self):
+        url = reverse("home")
+
+        responses = []
+
+        for i in range(16):
+            response = self.client.get(
+                url,
+                HTTP_USER_AGENT="test-agent",
+            )
+            responses.append(response.status_code)
+
+        assert all(code == 200 for code in responses[:15])
+
+        assert responses[15] == 429

vulnerabilities/throttling.py

@@ -51,6 +51,19 @@ def get_throttle_rate(self, tier):
             raise ImproperlyConfigured(msg)
 
 
+class AnonUserUIThrottle(UserRateThrottle):
+    scope = "ui"
+
+    def allow_request(self, request, view):
+        self.rate = self.THROTTLE_RATES.get("ui")
+        self.num_requests, self.duration = self.parse_rate(self.rate)
+        return super().allow_request(request, view)
+
+    def get_cache_key(self, request, view):
+        ident = self.get_ident(request)
+        return f"throttle_ui_{ident}"
+
+
 def throttled_exception_handler(exception, context):
     """
     Return this response whenever a request has been throttled

vulnerabilities/views.py

@@ -20,6 +20,7 @@
 from django.db.models import Exists
 from django.db.models import OuterRef
 from django.db.models import Prefetch
+from django.http import HttpResponse
 from django.http.response import Http404
 from django.shortcuts import get_object_or_404
 from django.shortcuts import redirect
@@ -47,6 +48,7 @@
 from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline
 from vulnerabilities.severity_systems import EPSS
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
+from vulnerabilities.throttling import AnonUserUIThrottle
 from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS
 from vulnerabilities.utils import get_advisories_from_groups
 from vulnerabilities.utils import merge_and_save_grouped_advisories
@@ -56,7 +58,47 @@
 PAGE_SIZE = 10
 
 
-class PackageSearch(ListView):
+class VulnerableCodeView(View):
+    """
+    Base ListView for VulnerableCode views that includes throttling.
+    """
+
+    throttle_classes = [AnonUserUIThrottle]
+
+    def dispatch(self, request, *args, **kwargs):
+        throttle = AnonUserUIThrottle()
+
+        if not throttle.allow_request(request, self):
+            return HttpResponse("Rate limit exceeded", status=429)
+
+        return super().dispatch(request, *args, **kwargs)
+
+
+class VulnerableCodeDetailView(DetailView, VulnerableCodeView):
+    """
+    Base DetailView for VulnerableCode views that includes throttling.
+    """
+
+    pass
+
+
+class VulnerableCodeListView(ListView, VulnerableCodeView):
+    """
+    Base ListView for VulnerableCode views that includes throttling.
+    """
+
+    pass
+
+
+class VulnerableCodeCreateView(generic.CreateView, VulnerableCodeView):
+    """
+    Base CreateView for VulnerableCode views that includes throttling.
+    """
+
+    pass
+
+
+class PackageSearch(VulnerableCodeListView):
     model = models.Package
     template_name = "packages.html"
     ordering = ["type", "namespace", "name", "version"]
@@ -84,7 +126,7 @@ def get_queryset(self, query=None):
         )
 
 
-class VulnerabilitySearch(ListView):
+class VulnerabilitySearch(VulnerableCodeListView):
     model = models.Vulnerability
     template_name = "vulnerabilities.html"
     ordering = ["vulnerability_id"]
@@ -102,7 +144,7 @@ def get_queryset(self, query=None):
         return self.model.objects.search(query=query).with_package_counts()
 
 
-class PackageDetails(DetailView):
+class PackageDetails(VulnerableCodeDetailView):
     model = models.Package
     template_name = "package_details.html"
     slug_url_kwarg = "purl"
@@ -143,7 +185,7 @@ def get_object(self, queryset=None):
         return package
 
 
-class PackageSearchV2(ListView):
+class PackageSearchV2(VulnerableCodeListView):
     model = models.PackageV2
     template_name = "packages_v2.html"
     ordering = ["type", "namespace", "name", "version"]
@@ -166,7 +208,7 @@ def get_queryset(self, query=None):
         return self.model.objects.search(query).prefetch_related().with_is_vulnerable()
 
 
-class AffectedByAdvisoriesListView(ListView):
+class AffectedByAdvisoriesListView(VulnerableCodeListView):
     model = models.AdvisoryV2
     template_name = "affected_by_advisories.html"
     paginate_by = PAGE_SIZE
@@ -187,7 +229,7 @@ def get_queryset(self):
         )
 
 
-class FixingAdvisoriesListView(ListView):
+class FixingAdvisoriesListView(VulnerableCodeListView):
     model = models.AdvisoryV2
     template_name = "fixing_advisories.html"
     paginate_by = PAGE_SIZE
@@ -201,7 +243,7 @@ def get_queryset(self):
         )
 
 
-class PackageV2Details(DetailView):
+class PackageV2Details(VulnerableCodeDetailView):
     model = models.PackageV2
     template_name = "package_details_v2.html"
     slug_url_kwarg = "purl"
@@ -439,7 +481,7 @@ def get_fixed_package_details(package):
     return fixed_pkg_details
 
 
-class VulnerabilityDetails(DetailView):
+class VulnerabilityDetails(VulnerableCodeDetailView):
     model = models.Vulnerability
     template_name = "vulnerability_details.html"
     slug_url_kwarg = "vulnerability_id"
@@ -543,7 +585,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class AdvisoryDetails(DetailView):
+class AdvisoryDetails(VulnerableCodeDetailView):
     model = models.AdvisoryV2
     template_name = "advisory_detail.html"
     slug_url_kwarg = "avid"
@@ -717,7 +759,7 @@ def add_ssvc(ssvc):
         return context
 
 
-class HomePage(View):
+class HomePage(VulnerableCodeView):
     template_name = "index.html"
 
     def get(self, request):
@@ -730,7 +772,7 @@ def get(self, request):
         return render(request=request, template_name=self.template_name, context=context)
 
 
-class HomePageV2(View):
+class HomePageV2(VulnerableCodeView):
     template_name = "index_v2.html"
 
     def get(self, request):
@@ -770,7 +812,7 @@ def get(self, request):
 """
 
 
-class ApiUserCreateView(generic.CreateView):
+class ApiUserCreateView(VulnerableCodeCreateView):
     model = models.ApiUser
     form_class = ApiUserCreationForm
     template_name = "api_user_creation_form.html"
@@ -800,7 +842,7 @@ def get_success_url(self):
         return reverse_lazy("api_user_request")
 
 
-class VulnerabilityPackagesDetails(DetailView):
+class VulnerabilityPackagesDetails(VulnerableCodeDetailView):
     """
     View to display all packages affected by or fixing a specific vulnerability.
     URL: /vulnerabilities/{vulnerability_id}/packages
@@ -851,7 +893,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class AdvisoryPackagesDetails(DetailView):
+class AdvisoryPackagesDetails(VulnerableCodeDetailView):
     """
     View to display all packages affected by or fixing a specific vulnerability.
     URL: /advisories/{id}/packages
@@ -902,7 +944,7 @@ def get_queryset(self):
         )
 
 
-class PipelineScheduleListView(ListView, FormMixin):
+class PipelineScheduleListView(VulnerableCodeListView, FormMixin):
     model = PipelineSchedule
     context_object_name = "schedule_list"
     template_name = "pipeline_dashboard.html"
@@ -926,7 +968,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class PipelineRunListView(ListView):
+class PipelineRunListView(VulnerableCodeListView):
     model = PipelineRun
     context_object_name = "run_list"
     template_name = "pipeline_run_list.html"
@@ -952,7 +994,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-class PipelineRunDetailView(DetailView):
+class PipelineRunDetailView(VulnerableCodeDetailView):
     model = PipelineRun
     template_name = "pipeline_run_details.html"
     context_object_name = "run"