Merge pull request #2249 from aboutcode-org/prep_v38.1.0

Prepare v38.1.0 for release

Author: TG1999

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Release notes
 =============
 
+Version v38.1.0
+---------------------
+
+- Throttle UI to 15 requests per minute to avoid abuse and improve performance.
+- Handle errors in unfurl_version_range pipeline.
+- Remove Todo pipeline from v1 pipelines.
+- Add openAPI documentation for Package and Advisory viewset.
+
 Version v38.0.0
 ---------------------
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 38.0.0
+version = 38.1.0
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

vulnerabilities/api_v3.py

@@ -14,6 +14,7 @@
 from django.db.models import OuterRef
 from django.db.models import Prefetch
 from django_filters import rest_framework as filters
+from drf_spectacular.utils import extend_schema
 from packageurl import PackageURL
 from rest_framework import serializers
 from rest_framework import viewsets
@@ -422,6 +423,7 @@ class PackageV3ViewSet(viewsets.GenericViewSet):
     filter_backends = [filters.DjangoFilterBackend]
     throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle]
 
+    @extend_schema(request=PackageQuerySerializer)
     def create(self, request, *args, **kwargs):
         serializer = PackageQuerySerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
@@ -528,8 +530,9 @@ class AdvisoryV3ViewSet(viewsets.GenericViewSet):
     filter_backends = [filters.DjangoFilterBackend]
     throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle]
 
+    @extend_schema(request=AdvisoryQuerySerializer)
     def create(self, request, *args, **kwargs):
-        serializer = PackageQuerySerializer(data=request.data)
+        serializer = AdvisoryQuerySerializer(data=request.data)
         serializer.is_valid(raise_exception=True)
 
         purls = serializer.validated_data["purls"]

vulnerabilities/importers/__init__.py

@@ -21,7 +21,6 @@
 from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
-from vulnerabilities.importers import openssl
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
 from vulnerabilities.importers import project_kb_msr2019
@@ -38,7 +37,6 @@
 from vulnerabilities.pipelines import gitlab_importer
 from vulnerabilities.pipelines import nginx_importer
 from vulnerabilities.pipelines import npm_importer
-from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
 from vulnerabilities.pipelines import pysec_importer
 from vulnerabilities.pipelines.v2_importers import alpine_linux_importer as alpine_linux_importer_v2
@@ -118,7 +116,6 @@
         retiredotnet_importer_v2.RetireDotnetImporterPipeline,
         ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline,
         alpine_linux_importer_v2.AlpineLinuxImporterPipeline,
-        nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,
         gitlab_importer.GitLabImporterPipeline,
         github_osv.GithubOSVImporter,
@@ -136,7 +133,6 @@
         alpine_linux_importer.AlpineLinuxImporterPipeline,
         ruby.RubyImporter,
         apache_kafka.ApacheKafkaImporter,
-        openssl.OpensslImporter,
         openssl_importer_v2.OpenSSLImporterPipeline,
         redhat.RedhatImporter,
         archlinux.ArchlinuxImporter,

vulnerabilities/improvers/__init__.py

@@ -10,7 +10,6 @@
 from vulnerabilities.improvers import valid_versions
 from vulnerabilities.improvers import vulnerability_status
 from vulnerabilities.pipelines import add_cvss31_to_CVEs
-from vulnerabilities.pipelines import compute_advisory_todo
 from vulnerabilities.pipelines import compute_package_risk
 from vulnerabilities.pipelines import compute_package_version_rank
 from vulnerabilities.pipelines import enhance_with_exploitdb
@@ -70,7 +69,6 @@
         compute_package_risk_v2.ComputePackageRiskPipeline,
         compute_version_rank_v2.ComputeVersionRankPipeline,
         unfurl_version_range_v2.UnfurlVersionRangePipeline,
-        compute_advisory_todo.ComputeToDo,
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
         group_advisories_for_packages.GroupAdvisoriesForPackages,

vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py

@@ -52,7 +52,7 @@ def unfurl_version_range(self):
             if purl.type not in RANGE_CLASS_BY_SCHEMES:
                 continue
 
-            versions = get_purl_versions(purl, cached_versions)
+            versions = get_purl_versions(purl, cached_versions) or []
             affected_purls = get_affected_purls(
                 versions=versions,
                 affecting_vers=impact.affecting_vers,
@@ -79,6 +79,8 @@ def get_affected_purls(versions, affecting_vers, base_purl, logger):
     version_class = affecting_version_range.version_class
 
     try:
+        if not versions:
+            return []
         versions = [version_class(v) for v in versions]
     except Exception as e:
         logger(
@@ -107,8 +109,10 @@ def get_affected_purls(versions, affecting_vers, base_purl, logger):
 
 def get_purl_versions(purl, cached_versions):
     if not purl in cached_versions:
-        cached_versions[purl] = get_versions(purl)
-    return cached_versions[purl]
+        purls = get_versions(purl)
+        if purls is not None:
+            cached_versions[purl] = purls
+    return cached_versions.get(purl) or []
 
 
 def bulk_create_with_m2m(purls, impact, relation, logger):

vulnerabilities/tests/test_view.py

@@ -11,8 +11,10 @@
 import time
 
 import pytest
+from django.core.cache import cache
 from django.test import Client
 from django.test import TestCase
+from django.urls import reverse
 from packageurl import PackageURL
 from univers import versions
 
@@ -330,3 +332,33 @@ def test_aggregate_fixed_and_affected_packages(self):
             end_time = time.time()
             assert end_time - start_time < 0.05
             self.assertEqual(response.status_code, 200)
+
+
+class ThrottleTestCase(TestCase):
+    def setUp(self):
+        self.client = Client()
+        cache.clear()
+
+    def test_throttle_after_15_requests(self):
+        url = reverse("home")
+
+        responses = []
+
+        for i in range(16):
+            response = self.client.get(
+                url,
+                HTTP_USER_AGENT="test-agent",
+            )
+            responses.append(response.status_code)
+
+        assert all(code == 200 for code in responses[:15])
+
+        assert responses[15] == 429
+
+        url = reverse("package_search")
+
+        response = self.client.get(
+            url,
+            HTTP_USER_AGENT="test-agent",
+        )
+        assert response.status_code == 429

vulnerabilities/throttling.py

@@ -51,6 +51,19 @@ def get_throttle_rate(self, tier):
             raise ImproperlyConfigured(msg)
 
 
+class AnonUserUIThrottle(UserRateThrottle):
+    scope = "ui"
+
+    def allow_request(self, request, view):
+        self.rate = self.THROTTLE_RATES.get("ui")
+        self.num_requests, self.duration = self.parse_rate(self.rate)
+        return super().allow_request(request, view)
+
+    def get_cache_key(self, request, view):
+        ident = self.get_ident(request)
+        return f"throttle_ui_{ident}"
+
+
 def throttled_exception_handler(exception, context):
     """
     Return this response whenever a request has been throttled