refactor: Simplify v2 recommender

openshift/template.yaml

@@ -56,6 +56,8 @@ objects:
             value: f8a-pypi-insights
           - name: GOLANG_SERVICE_HOST
             value: f8a-golang-insights
+          - name: MAVEN_SERVICE_HOST
+            value: f8a-hpf-insights-maven
           - name: HPF_SERVICE_HOST
             value: f8a-hpf-insights
           - name: SERVICE_PORT

scripts/entrypoint.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/bash
 
 # Start API backbone service with time out
-gunicorn --pythonpath /src/ -b 0.0.0.0:$API_BACKBONE_SERVICE_PORT -t $API_BACKBONE_SERVICE_TIMEOUT -k $CLASS_TYPE -w $NUMBER_WORKER_PROCESS rest_api:app
+gunicorn --pythonpath /src/ -b 0.0.0.0:$API_BACKBONE_SERVICE_PORT -t $API_BACKBONE_SERVICE_TIMEOUT -k $CLASS_TYPE -w $NUMBER_WORKER_PROCESS rest_api:app --reload --log-level $FLASK_LOGGING_LEVEL

src/rest_api.py

@@ -1,43 +1,28 @@
 """Implementation of the REST API for the backbone service."""
 
-import os
-import logging
 import flask
+import logging
+import os
 import time
+
 from f8a_worker.setup_celery import init_selinon
 from flask import Flask, request
 from flask_cors import CORS
 from raven.contrib.flask import Sentry
 
 from src.recommender import RecommendationTask as RecommendationTaskV1
 from src.stack_aggregator import StackAggregator as StackAggregatorV1
+from src.utils import push_data, total_time_elapsed
 from src.v2.recommender import RecommendationTask as RecommendationTaskV2
 from src.v2.stack_aggregator import StackAggregator as StackAggregatorV2
-from src.utils import push_data, total_time_elapsed, get_time_delta
-
-
-def setup_logging(flask_app):
-    """Perform the setup of logging (file, log level) for this application."""
-    if not flask_app.debug:
-        handler = logging.StreamHandler()
-        handler.setFormatter(logging.Formatter(
-            '[%(asctime)s] %(levelname)s in %(pathname)s:%(lineno)d: %(message)s'))
-        log_level = os.environ.get('FLASK_LOGGING_LEVEL', logging.getLevelName(logging.WARNING))
-        handler.setLevel(log_level)
-
-        flask_app.logger.addHandler(handler)
-        flask_app.config['LOGGER_HANDLER_POLICY'] = 'never'
-        flask_app.logger.setLevel(logging.DEBUG)
 
+logger = logging.getLogger(__name__)
 
 app = Flask(__name__)
-setup_logging(app)
 CORS(app)
 SENTRY_DSN = os.environ.get("SENTRY_DSN", "")
 sentry = Sentry(app, dsn=SENTRY_DSN, logging=True, level=logging.ERROR)
 
-logger = logging.getLogger(__name__)
-
 init_selinon()
 
 
@@ -68,29 +53,25 @@ def _recommender(handler):
     }
 
     input_json = request.get_json()
-    if input_json and 'external_request_id' in input_json and input_json['external_request_id']:
-        external_request_id = input_json['external_request_id']
-        logger.info('%s recommender/ request with payload: %s', external_request_id, input_json)
-
-        try:
-            check_license = request.args.get('check_license', 'false') == 'true'
-            persist = request.args.get('persist', 'true') == 'true'
-            r = handler.execute(input_json, persist=persist,
-                                check_license=check_license)
-        except Exception as e:
-            r = {
-                'recommendation': 'unexpected error',
-                'external_request_id': input_json.get('external_request_id'),
-                'message': '%s' % e
-            }
-            metrics_payload['status_code'] = 400
-            logger.error('%s failed %s', external_request_id, r)
+    external_request_id = input_json['external_request_id']
+    logger.info('%s recommender/ request with payload: %s', external_request_id, input_json)
 
     try:
-        metrics_payload['value'] = get_time_delta(audit_data=r['result']['_audit'])
+        check_license = request.args.get('check_license', 'false') == 'true'
+        persist = request.args.get('persist', 'true') == 'true'
+        r = handler.execute(input_json, persist=persist,
+                            check_license=check_license)
+    except Exception as e:
+        r = {
+            'recommendation': 'unexpected error',
+            'external_request_id': input_json.get('external_request_id'),
+            'message': '%s' % e
+        }
+        metrics_payload['status_code'] = 400
+        metrics_payload['value'] = 0
         push_data(metrics_payload)
-    except KeyError:
-        pass
+        logger.error('%s failed %s', external_request_id, r)
+        raise e
 
     logger.info('%s took %0.2f seconds for _recommender',
                 external_request_id, time.time() - recommender_started_at)
@@ -114,38 +95,31 @@ def _stack_aggregator(handler):
         'status_code': 200
     }
 
-    if input_json and 'external_request_id' in input_json \
-            and input_json['external_request_id']:
-        external_request_id = input_json['external_request_id']
-        logger.info('%s stack_aggregator/ request with payload: %s',
-                    external_request_id, input_json)
-
-        try:
-            persist = request.args.get('persist', 'true') == 'true'
-            s = handler.execute(input_json, persist=persist)
-            if s is not None and s.get('result') and s.get('result').get('_audit'):
-                # Creating and Pushing Total Metrics Data to Accumulator
-                metrics_payload['value'] = total_time_elapsed(
-                    sa_audit_data=s['result']['_audit'],
-                    external_request_id=input_json['external_request_id'])
-                push_data(metrics_payload)
-
-        except Exception as e:
-            s = {
-                'stack_aggregator': 'unexpected error',
-                'external_request_id': input_json.get('external_request_id'),
-                'message': '%s' % e
-            }
-            metrics_payload['status_code'] = 400
-            logger.error('%s failed %s', external_request_id, s)
-
-        try:
-            # Pushing Individual Metrics Data to Accumulator
-            metrics_payload['value'] = get_time_delta(audit_data=s['result']['_audit'])
-            metrics_payload['endpoint'] = request.endpoint
+    external_request_id = input_json['external_request_id']
+    logger.info('%s stack_aggregator/ request with payload: %s',
+                external_request_id, input_json)
+
+    try:
+        persist = request.args.get('persist', 'true') == 'true'
+        s = handler.execute(input_json, persist=persist)
+        if s.get('result', {}).get('_audit'):
+            # Creating and Pushing Total Metrics Data to Accumulator
+            metrics_payload['value'] = total_time_elapsed(
+                sa_audit_data=s['result']['_audit'],
+                external_request_id=input_json['external_request_id'])
             push_data(metrics_payload)
-        except KeyError:
-            pass
+    except Exception as e:
+        s = {
+            'stack_aggregator': 'unexpected error',
+            'external_request_id': input_json.get('external_request_id'),
+            'message': '%s' % e
+        }
+        metrics_payload['status_code'] = 400
+        # Pushing Individual Metrics Data to Accumulator
+        metrics_payload['value'] = 0
+        push_data(metrics_payload)
+        logger.error('%s failed %s', external_request_id, s)
+        raise e
 
     logger.info('%s took %0.2f seconds for _stack_aggregators',
                 external_request_id, time.time() - stack_aggregator_started_at)

src/settings.py

@@ -5,11 +5,21 @@
 from pydantic import BaseSettings, HttpUrl
 
 
-# (fixme) Move all settings to read from here.
-class Settings(BaseSettings):
+class AggregatorSettings(BaseSettings):
     """Create Settings from env."""
 
-    snyk_package_url_format: HttpUrl = 'https://snyk.io/vuln/{ecosystem}:{package}'
-    snyk_signin_url: HttpUrl = 'https://snyk.io/login'
+    snyk_package_url_format: HttpUrl = "https://snyk.io/vuln/{ecosystem}:{package}"
+    snyk_signin_url: HttpUrl = "https://snyk.io/login"
     snyk_ecosystem_map: Dict[str, str] = {"pypi": "pip"}
     disable_unknown_package_flow: bool = False
+
+
+class RecommenderSettings(BaseSettings):
+    """Create Recommender Settings from env."""
+
+    chester_service_host: str = "notset"
+    pypi_service_host: str = "notset"
+    maven_service_host: str = "notset"
+    service_port: int = 6006
+    unknown_packages_threshold: float = 0.3
+    max_companion_packages: int = 5

src/stack_aggregator.py

@@ -12,7 +12,7 @@
 import requests
 import copy
 from collections import defaultdict
-from src.settings import Settings
+from src.settings import AggregatorSettings
 from src.utils import (select_latest_version, server_create_analysis, LICENSE_SCORING_URL_REST,
                        post_http_request, GREMLIN_SERVER_URL_REST, persist_data_in_db,
                        GREMLIN_QUERY_SIZE, format_date)
@@ -663,7 +663,7 @@ def execute(aggregated=None, persist=True):
                          'result': stack_data}
         # Ingestion of Unknown dependencies
         logger.info("Unknown ingestion flow process initiated.")
-        if Settings().disable_unknown_package_flow:
+        if AggregatorSettings().disable_unknown_package_flow:
             logger.warning('Skipping unknown flow %s', unknown_dep_list)
         else:
             try:

src/utils.py

@@ -324,21 +324,21 @@ def post_http_request(url: str, payload: Dict):
 
 def post_gremlin(query: str, bindings: Dict = None) -> Dict:
     """Post the given query and bindings to gremlin endpoint."""
+    payload = {
+        'gremlin': query,
+    }
+    if bindings:
+        payload['bindings'] = bindings
     try:
-        payload = {
-            'gremlin': query,
-        }
-        if bindings:
-            payload['bindings'] = bindings
         response = get_session_retry().post(url=GREMLIN_SERVER_URL_REST, json=payload)
         response.raise_for_status()
-        return response.json()
     except Exception as e:
         logger.error(traceback.format_exc())
-        logger.error(
-            "HTTP error {code}. Error retrieving data for {query}.".format(
-                code=response.status_code, query=payload))
+        logger.error("HTTP error %d. Error retrieving data for %s.",
+                     response.status_code, payload)
         raise GremlinExeception from e
+    else:
+        return response.json()
 
 
 def get_response_data(json_response, data_default):

src/v2/models.py

@@ -120,7 +120,6 @@ class UsedByItem(BaseModel):  # noqa: D101
 
 class GitHubDetails(BaseModel):  # noqa: D101
     watchers: Optional[str] = None
-    first_release_date: Optional[str] = None
     total_releases: Optional[str] = None
     issues: Optional[Dict[str, Any]] = None
     pull_requests: Optional[Dict[str, Any]] = None
@@ -222,13 +221,14 @@ class StackRecommendationResult(BaseModel):  # noqa: D101
     registration_status: str
     recommendation_status: 'RecommendationStatus' = 'success'
     companion: List['RecommendedPackageData']
-    usage_outliers: List[Dict[str, Any]]
+    usage_outliers: List[Dict[str, Any]] = []
 
 
 class RecommenderRequest(StackAggregatorRequest):  # noqa: D101
     pass
 
 
 Package.update_forward_refs()
+PackageDetails.update_forward_refs()
 PackageDataWithVulnerabilities.update_forward_refs()
 RecommendedPackageData.update_forward_refs()

src/v2/openapi.yaml

@@ -360,8 +360,6 @@ components:
         watchers:
           type: string
           example: "100K"
-        first_release_date:
-          type: string
         total_releases:
           type: string
           example: "12"
@@ -487,7 +485,6 @@ components:
         - recommendation_status
         - companion
         - manifest_file_path
-        - usage_outliers
       properties:
         _audit:
           $ref: '#/components/schemas/Audit'
@@ -513,11 +510,6 @@ components:
         manifest_file_path:
           type: string
           example: pom.xml
-        usage_outliers:
-          type: array
-          items:
-            type: object
-            properties: {}
       description: Application stack recommendations
 
     Audit: