Merge pull request GreedyBear-Project#1203 from GreedyBear-Project/develop

3.3.1

Author: regulartim

.github/FUNDING.yml

@@ -1,2 +1 @@
 open_collective: intelowl-project
-github: intelowlproject

README.md

@@ -54,10 +54,12 @@ To install it locally, Please refer to our [installation guide](https://intelowl
 
 Thanks to [The Honeynet Project](https://www.honeynet.org) we are providing free public feeds available [here](https://greedybear.honeynet.org).
 
-#### DigitalOcean
+#### Google Summer of Code
+<a href="https://summerofcode.withgoogle.com/"> <img style="border: 0.2px solid black" width=150 height=89 src="static/gsoc_logo.png" alt="GSoC logo"> </a>
 
-In 2022 we joined the official [DigitalOcean Open Source Program](https://www.digitalocean.com/open-source?utm_medium=opensource&utm_source=IntelOwl).
+In 2026 we started participating to the [Google Summer of Code](https://summerofcode.withgoogle.com/) (GSoC)!
 
+If you are interested in participating in the next Google Summer of Code, check all the info available in the [dedicated repository](https://github.com/intelowlproject/gsoc)!
 
 ## Maintainers and Key Contributors
 

api/serializers.py

@@ -6,15 +6,15 @@
 from rest_framework import serializers
 
 from greedybear.consts import REGEX_DOMAIN
-from greedybear.models import IOC, GeneralHoneypot, Sensor, Tag
+from greedybear.models import IOC, Honeypot, Sensor, Tag
 from greedybear.utils import is_ip_address
 
 logger = logging.getLogger(__name__)
 
 
-class GeneralHoneypotSerializer(serializers.ModelSerializer):
+class HoneypotSerializer(serializers.ModelSerializer):
     class Meta:
-        model = GeneralHoneypot
+        model = Honeypot
 
     def to_representation(self, value):
         return value.name
@@ -33,7 +33,7 @@ class Meta:
 
 
 class IOCSerializer(serializers.ModelSerializer):
-    general_honeypot = GeneralHoneypotSerializer(many=True, read_only=True)
+    general_honeypot = HoneypotSerializer(many=True, read_only=True, source="honeypots")
     tags = TagSerializer(many=True, read_only=True)
     sensors = SensorSerializer(many=True, read_only=True)
 

api/views/__init__.py

@@ -2,7 +2,7 @@
 from api.views.cowrie_session import *
 from api.views.enrichment import *
 from api.views.feeds import *
-from api.views.general_honeypot import *
 from api.views.health import *
+from api.views.honeypots import *
 from api.views.news import *
 from api.views.statistics import *

api/views/health.py

@@ -15,7 +15,7 @@
     IOC,
     CowrieSession,
     FireHolList,
-    GeneralHoneypot,
+    Honeypot,
     MassScanner,
     TorExitNode,
 )
@@ -71,8 +71,8 @@ def get_observables_overview(last_24h):
     )
 
     honeypot_stats = {
-        "total": GeneralHoneypot.objects.count(),
-        "active": GeneralHoneypot.objects.filter(active=True).count(),
+        "total": Honeypot.objects.count(),
+        "active": Honeypot.objects.filter(active=True).count(),
     }
 
     threat_list_stats = {

api/views/general_honeypot.py api/views/honeypots.pyapi/views/general_honeypot.py renamed to api/views/honeypots.py

@@ -6,7 +6,7 @@
 from rest_framework.response import Response
 
 from greedybear.consts import GET
-from greedybear.models import GeneralHoneypot
+from greedybear.models import Honeypot
 
 logger = logging.getLogger(__name__)
 
@@ -23,14 +23,14 @@ def general_honeypot_list(request):
         Response: A JSON response containing the list of general honeypots.
     """
 
-    logger.info(f"Requested general honeypots list from {request.user}.")
+    logger.info(f"Requested honeypots list from {request.user}.")
     active = request.query_params.get("onlyActive")
     honeypots = []
-    general_honeypots = GeneralHoneypot.objects.all()
+    honeypot_objs = Honeypot.objects.all()
     if active == "true":
-        general_honeypots = general_honeypots.filter(active=True)
-        logger.info(f"Requested only active general honeypots from {request.user}")
-    honeypots.extend([hp.name for hp in general_honeypots])
+        honeypot_objs = honeypot_objs.filter(active=True)
+        logger.info(f"Requested only active honeypots from {request.user}")
+    honeypots.extend([hp.name for hp in honeypot_objs])
 
-    logger.info(f"General honeypots: {honeypots} given back to user {request.user}")
+    logger.info(f"Honeypots: {honeypots} given back to user {request.user}")
     return Response(honeypots)

api/views/statistics.py

@@ -10,7 +10,7 @@
 from rest_framework.decorators import action
 from rest_framework.response import Response
 
-from greedybear.models import IOC, GeneralHoneypot, Statistics, ViewType
+from greedybear.models import IOC, Honeypot, Statistics, ViewType
 
 logger = logging.getLogger(__name__)
 
@@ -92,7 +92,7 @@ def countries(self, request):
         qs = (
             IOC.objects.filter(last_seen__gte=delta)
             .exclude(attacker_country="")
-            .filter(general_honeypot__active=True)
+            .filter(honeypots__active=True)
             .values("attacker_country")
             .annotate(count=Count("id", distinct=True))
             .order_by("-count")
@@ -103,7 +103,7 @@ def countries(self, request):
     @action(detail=False, methods=["get"])
     def feeds_types(self, request):
         """
-        Retrieve statistics for different types of feeds using GeneralHoneypot M2M relationship.
+        Retrieve statistics for different types of feeds using Honeypot M2M relationship.
 
         Args:
             request: The incoming request object.
@@ -113,10 +113,10 @@ def feeds_types(self, request):
         """
         # Build annotations for each active general honeypot
         annotations = {}
-        general_honeypots = GeneralHoneypot.objects.all().filter(active=True)
-        for hp in general_honeypots:
+        honeypots = Honeypot.objects.all().filter(active=True)
+        for hp in honeypots:
             # Use M2M relationship instead of boolean fields
-            annotations[hp.name] = Count("name", distinct=True, filter=Q(general_honeypot__name__iexact=hp.name))
+            annotations[hp.name] = Count("name", distinct=True, filter=Q(honeypots__name__iexact=hp.name))
         return self.__aggregation_response_static_ioc(annotations)
 
     def __aggregation_response_static_statistics(self, annotations: dict) -> Response:
@@ -147,7 +147,7 @@ def __aggregation_response_static_ioc(self, annotations: dict) -> Response:
 
         qs = (
             IOC.objects.filter(last_seen__gte=delta)
-            .exclude(general_honeypot__active=False)
+            .exclude(honeypots__active=False)
             .annotate(date=Trunc("last_seen", basis))
             .values("date")
             .annotate(**annotations)

api/views/utils.py

@@ -1,14 +1,16 @@
 # This file is a part of GreedyBear https://github.com/honeynet/GreedyBear
 # See the file 'LICENSE' for copying permission.
 import csv
+import hashlib
 import logging
+import urllib.parse
 from datetime import datetime, timedelta
 
 import feedparser
 import requests
 from django.conf import settings
 from django.contrib.postgres.aggregates import ArrayAgg
-from django.core.cache import cache
+from django.core.cache import cache, caches
 from django.db.models import Count, F, Max, Min, Q, Sum, Value
 from django.db.models.functions import JSONObject
 from django.http import HttpResponse, HttpResponseBadRequest, StreamingHttpResponse
@@ -19,7 +21,7 @@
 from api.serializers import FeedsRequestSerializer, parse_feed_types
 from greedybear.consts import CACHE_KEY_GREEDYBEAR_NEWS, CACHE_TIMEOUT_SECONDS, RSS_FEED_URL
 from greedybear.enums import IpReputation
-from greedybear.models import IOC, GeneralHoneypot, Statistics
+from greedybear.models import IOC, Honeypot, Statistics
 from greedybear.utils import is_ip_address, is_valid_domain
 
 logger = logging.getLogger(__name__)
@@ -144,8 +146,8 @@ def get_valid_feed_types() -> frozenset[str]:
     Returns:
         frozenset[str]: An immutable set of valid feed type strings
     """
-    general_honeypots = GeneralHoneypot.objects.filter(active=True)
-    feed_types = ["all"] + [hp.name.lower() for hp in general_honeypots]
+    honeypots = Honeypot.objects.filter(active=True)
+    feed_types = ["all"] + [hp.name.lower() for hp in honeypots]
     return frozenset(feed_types)
 
 
@@ -226,13 +228,13 @@ def get_queryset(request, feed_params, valid_feed_types, is_aggregated=False, se
     if "all" not in feed_params.feed_types:
         type_filter = Q()
         for ft in feed_params.feed_types:
-            type_filter |= Q(general_honeypot__name__iexact=ft)
+            type_filter |= Q(honeypots__name__iexact=ft)
         iocs = iocs.filter(type_filter)
 
     # aggregated feeds calculate metrics differently and need all rows to be accurate.
     if not is_aggregated:
-        iocs = iocs.filter(general_honeypot__active=True)
-        iocs = iocs.annotate(honeypots=ArrayAgg("general_honeypot__name", distinct=True))
+        iocs = iocs.filter(honeypots__active=True)
+        iocs = iocs.annotate(honeypot_names=ArrayAgg("honeypots__name", distinct=True))
         # Only annotate tags metadata when the response format needs it (e.g. JSON),
         # to avoid unnecessary joins and aggregation work for txt/csv feeds.
         if getattr(feed_params, "format", "").lower() == "json":
@@ -315,7 +317,7 @@ def feeds_response(request=None, iocs=None, feed_params=None, valid_feed_types=N
                 "login_attempts",
                 "recurrence_probability",
                 "expected_interactions",
-                "honeypots",  # used to build feed_type; removed from response
+                "honeypot_names",  # used to build feed_type; removed from response
                 "destination_ports",  # used to calculate destination_port_count
                 "attacker_country",
                 "autonomous_system",
@@ -344,7 +346,7 @@ def feeds_response(request=None, iocs=None, feed_params=None, valid_feed_types=N
             else:
                 iocs_iter = iocs.values(*required_fields).iterator(chunk_size=2000)
             for ioc in iocs_iter:
-                ioc_feed_type = [hp.lower() for hp in ioc.get("honeypots", []) if hp]
+                ioc_feed_type = [hp.lower() for hp in ioc.get("honeypot_names", []) if hp]
 
                 data_ = ioc | {
                     "first_seen": ioc["first_seen"].strftime("%Y-%m-%d"),
@@ -358,7 +360,7 @@ def feeds_response(request=None, iocs=None, feed_params=None, valid_feed_types=N
                 if not verbose:
                     data_.pop("destination_ports", None)
                 data_.pop("autonomous_system", None)
-                data_.pop("honeypots", None)
+                data_.pop("honeypot_names", None)
                 data_.pop("id", None)
 
                 json_list.append(data_)
@@ -386,7 +388,7 @@ def feeds_response(request=None, iocs=None, feed_params=None, valid_feed_types=N
                 "first_seen",
                 "last_seen",
                 "recurrence_probability",
-                "honeypots",
+                "honeypot_names",
                 "ip_reputation",
             }
             # Fetch fields from database
@@ -416,7 +418,7 @@ def feeds_response(request=None, iocs=None, feed_params=None, valid_feed_types=N
                 confidence = 90
 
                 # Labels
-                labels = [hp.lower() for hp in ioc.get("honeypots", []) if hp]
+                labels = [hp.lower() for hp in ioc.get("honeypot_names", []) if hp]
                 if ioc.get("ip_reputation"):
                     labels.append(ioc["ip_reputation"])
 
@@ -446,15 +448,33 @@ def feeds_response(request=None, iocs=None, feed_params=None, valid_feed_types=N
 
 def asn_aggregated_queryset(iocs_qs, request, feed_params):
     """
-    Perform DB-level aggregation grouped by ASN.
+    Retrieve ASN aggregation data. Caches the heavy aggregation query
+    since the data only updates during the extraction cronjob.
 
     Args
         iocs_qs (QuerySet): Filtered IOC queryset from get_queryset;
         request (Request): The API request object;
         feed_params (FeedRequestParams): Validated parameter object
 
-    Returns: A values-grouped queryset with annotated  metrics and honeypot arrays.
+    Returns: A list of dicts with aggregated metrics and honeypot arrays per ASN.
     """
+
+    # Build reliable cache key from query params
+    sorted_params = sorted(request.query_params.lists())
+    params_string = urllib.parse.urlencode(sorted_params, doseq=True)
+    param_hash = hashlib.sha256(params_string.encode("utf-8")).hexdigest()
+
+    # To prevent per-worker continuous RAM bloat, use the shared DB-backed cache
+    # instead of the default LocMemCache, since the JSON response size can be large.
+    # The extraction pipeline invalidates this cache by bumping the version counter.
+    shared_cache = caches["django-q"]
+    version = shared_cache.get("asn_feeds_version", 1)
+    cache_key = f"asn_feeds_v{version}_{param_hash}"
+
+    cached_result = shared_cache.get(cache_key)
+    if cached_result is not None:
+        return cached_result
+
     asn_filter = request.query_params.get("asn")
     if asn_filter:
         iocs_qs = iocs_qs.filter(autonomous_system__asn=asn_filter)
@@ -480,31 +500,35 @@ def asn_aggregated_queryset(iocs_qs, request, feed_params):
             first_seen=Min("first_seen"),
             last_seen=Max("last_seen"),
         )
-        .order_by(ordering)
     )
+    numeric_agg = numeric_agg.order_by(ordering)
 
+    # Honeypot names still require a lightweight aggregation because
+    # they depend on the active flag which can change independently.
     honeypot_agg = (
         iocs_qs.exclude(autonomous_system__isnull=True)
-        .filter(general_honeypot__active=True)
+        .filter(honeypots__active=True)
         .values(asn=F("autonomous_system__asn"))
         .annotate(
-            honeypots=ArrayAgg(
-                "general_honeypot__name",
+            honeypot_names=ArrayAgg(
+                "honeypots__name",
                 distinct=True,
             )
         )
     )
 
-    hp_lookup = {row["asn"]: row["honeypots"] or [] for row in honeypot_agg}
+    hp_lookup = {row["asn"]: row["honeypot_names"] or [] for row in honeypot_agg}
 
-    # merging numeric aggregate with honeypot names for each asn
     result = []
     for row in numeric_agg:
         asn = row["asn"]
         row_dict = dict(row)
         row_dict["honeypots"] = sorted(hp_lookup.get(asn, []))
         result.append(row_dict)
 
+    # Set cache with a 60-minute timeout (max extraction interval length) to prevent memory bloat
+    shared_cache.set(cache_key, result, timeout=3600)
+
     return result
 
 

configuration/gunicorn/config.py

@@ -1,10 +1,10 @@
-import multiprocessing
+import os
 
 # Server socket
 bind = "unix:/run/gunicorn/main.sock"
 
 # Worker processes
-workers = 2 * multiprocessing.cpu_count() + 1
+workers = 2 * len(os.sched_getaffinity(0)) + 1
 max_requests = 1000
 max_requests_jitter = 50
 

docker/Dockerfile

@@ -37,30 +37,37 @@ ENV UV_PROJECT_ENVIRONMENT=/usr/local
 
 WORKDIR $APP_ROOT
 
-# Install runtime dependencies
-#  - libgomp1 is required for model training
-#  - curl is used for healthcheck
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libgomp1 curl gosu \
+# Layer 1: stable runtime OS deps — cached across pyproject.toml/uv.lock changes.
+#  libgomp1: model training; curl: healthcheck; gosu: entrypoint privilege drop
+#  libpq5: runtime shared library required by the psycopg[c] C extension
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends libgomp1 curl gosu libpq5 \
     && rm -rf /var/lib/apt/lists/*
 
-# Install python packages
+# Layer 2: Python packages — only re-runs when pyproject.toml/uv.lock change.
+#  Build-only deps (gcc, python3-dev, libpq-dev) compile the psycopg[c] C
+#  extension and are purged in the same layer to keep the final image lean.
 COPY pyproject.toml uv.lock ./
-RUN uv sync --no-dev --locked
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends gcc python3-dev libpq-dev \
+    && uv sync --no-dev --locked \
+    && uv cache clean \
+    && apt-get purge -y gcc python3-dev libpq-dev \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/*
 
 # Copy files
 COPY . $APP_ROOT
 COPY --from=frontend-build /app/build /var/www/reactapp
 
-# separation is required to avoid to re-execute os installation in case of change of python requirements
+# Set up log directories, fix permissions, and remove frontend source (served from /var/www/reactapp)
 RUN mkdir -p ${LOG_PATH}/django ${LOG_PATH}/gunicorn \
     && touch ${LOG_PATH}/django/api.log ${LOG_PATH}/django/api_errors.log \
     && touch ${LOG_PATH}/django/greedybear.log ${LOG_PATH}/django/greedybear_errors.log \
     && touch ${LOG_PATH}/django/django_q.log ${LOG_PATH}/django/django_q_errors.log \
     && touch ${LOG_PATH}/django/django_errors.log ${LOG_PATH}/django/elasticsearch.log \
     && touch ${LOG_PATH}/django/authentication.log ${LOG_PATH}/django/authentication_errors.log \
     && mkdir -p ${APP_ROOT}/mlmodels \
-    && usermod -u 2000 www-data \
     && chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ ${APP_ROOT}/mlmodels/ \
     && rm -rf frontend/