chore: Unify internal service host config names (#269)

Author: arajkumar

openshift/template.yaml

@@ -26,10 +26,8 @@ objects:
         - command:
           - /bin/entrypoint.sh
           env:
-          - name: BAYESIAN_GREMLIN_HTTP_SERVICE_HOST
-            value: bayesian-gremlin-http
-          - name: BAYESIAN_GREMLIN_HTTP_SERVICE_PORT
-            value: "8182"
+          - name: GREMLIN_URL
+            value: "http://bayesian-gremlin-http:8182"
           - name: DEPLOYMENT_PREFIX
             valueFrom:
               configMapKeyRef:
@@ -46,22 +44,14 @@ objects:
             value: bayesian-pgbouncer
           - name: PGBOUNCER_SERVICE_PORT
             value: "5432"
-          - name: LICENSE_SERVICE_HOST
-            value: f8a-license-analysis
-          - name: LICENSE_SERVICE_PORT
-            value: "6162"
-          - name: CHESTER_SERVICE_HOST
-            value: f8a-npm-insights
-          - name: PYPI_SERVICE_HOST
-            value: f8a-pypi-insights
-          - name: GOLANG_SERVICE_HOST
-            value: f8a-golang-insights
-          - name: MAVEN_SERVICE_HOST
-            value: f8a-hpf-insights-maven
-          - name: HPF_SERVICE_HOST
-            value: f8a-hpf-insights
-          - name: SERVICE_PORT
-            value: "6006"
+          - name: LICENSE_ANALYSIS_BASE_URL
+            value: "http://f8a-license-analysis:6162"
+          - name: NPM_INSIGHTS_BASE_URL
+            value: "http://f8a-npm-insights:6006"
+          - name: PYPI_INSIGHTS_BASE_URL
+            value: "http://f8a-pypi-insights:6006"
+          - name: MAVEN_INSIGHTS_BASE_URL
+            value: "http://f8a-hpf-insights-maven:6006"
           - name: POSTGRESQL_DATABASE
             valueFrom:
               secretKeyRef:

src/recommender.py

@@ -70,8 +70,10 @@
 import re
 import logging
 
+from pydantic import AnyHttpUrl
+
+from src.settings import AGGREGATOR_SETTINGS, RECOMMENDER_SETTINGS, SETTINGS
 from src.utils import (create_package_dict, get_session_retry, select_latest_version,
-                       GREMLIN_SERVER_URL_REST, LICENSE_SCORING_URL_REST,
                        convert_version_to_proper_semantic, get_response_data,
                        version_info_tuple, persist_data_in_db, post_http_request)
 from src.stack_aggregator import extract_user_stack_package_licenses
@@ -112,7 +114,7 @@ def get_version_information(input_list, ecosystem):
         }
 
         # Query Gremlin with packages list to get their version information
-        gremlin_response = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+        gremlin_response = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
         if gremlin_response is None:
             return []
         response = get_response_data(gremlin_response, [{0: 0}])
@@ -289,7 +291,7 @@ class License:
     @staticmethod
     def invoke_license_analysis_service(user_stack_packages, alt_packages, comp_packages):
         """Pass given args to stack_license analysis."""
-        license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license"
+        license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license"
 
         payload = {
             "packages": user_stack_packages,
@@ -413,41 +415,20 @@ def set_valid_cooccurrence_probability(package_list=[]):
     return new_package_list
 
 
-class RecommendationTask:
-    """Recommendation task."""
+def _prepare_insights_url(base_url: AnyHttpUrl) -> AnyHttpUrl:
+    assert base_url
+    return "{url}/api/v1/companion_recommendation".format(url=base_url)
 
-    _analysis_name = 'recommendation_v2'
-    description = 'Get Recommendation'
-    kronos_ecosystems = ['maven']
-    chester_ecosystems = ['npm']
-    hpf_ecosystems = ['maven']
-    pypi_ecosystems = ['pypi']
-    golang_ecosystem = ['golang']
 
-    @staticmethod
-    def get_insights_url(payload):
-        """Get the insights url based on the ecosystem."""
-        if payload and 'ecosystem' in payload[0]:
-            if payload[0]['ecosystem'] in RecommendationTask.chester_ecosystems:
-                INSIGHTS_SERVICE_HOST = os.getenv("CHESTER_SERVICE_HOST")
-            elif payload[0]['ecosystem'] in RecommendationTask.pypi_ecosystems:
-                INSIGHTS_SERVICE_HOST = os.getenv("PYPI_SERVICE_HOST")
-            elif payload[0]['ecosystem'] in RecommendationTask.golang_ecosystem:
-                INSIGHTS_SERVICE_HOST = os.environ.get("GOLANG_SERVICE_HOST")
-            else:
-                INSIGHTS_SERVICE_HOST = os.getenv("HPF_SERVICE_HOST") + "-" + payload[0][
-                    'ecosystem']
+ECOSYSTEM_TO_INSIGHTS_URL = {
+    "pypi": _prepare_insights_url(RECOMMENDER_SETTINGS.pypi_insights_base_url),
+    "npm": _prepare_insights_url(RECOMMENDER_SETTINGS.npm_insights_base_url),
+    "maven": _prepare_insights_url(RECOMMENDER_SETTINGS.maven_insights_base_url),
+}
 
-            INSIGHTS_URL_REST = "http://{host}:{port}".format(host=INSIGHTS_SERVICE_HOST,
-                                                              port=os.getenv("SERVICE_PORT"))
 
-            insights_url = INSIGHTS_URL_REST + "/api/v1/companion_recommendation"
-
-            return insights_url
-
-        else:
-            logger.error('Payload information not passed in the call, Quitting! inights '
-                         'recommender\'s call')
+class RecommendationTask:
+    """Recommendation task."""
 
     @staticmethod
     def call_insights_recommender(payload):
@@ -459,7 +440,8 @@ def call_insights_recommender(payload):
         try:
             # TODO remove hardcodedness for payloads with multiple ecosystems
 
-            insights_url = RecommendationTask.get_insights_url(payload)
+            insights_url = ECOSYSTEM_TO_INSIGHTS_URL.get(payload[0]['ecosystem'], None)
+            assert insights_url
             response = get_session_retry().post(insights_url, json=payload)
 
             if response.status_code != 200:
@@ -519,13 +501,6 @@ def execute(self, arguments=None, persist=True, check_license=False):
                 'comp_package_count_threshold': int(os.environ.get(
                     'MAX_COMPANION_PACKAGES', 5))
             }
-            if details['ecosystem'] in self.kronos_ecosystems:
-                insights_payload.update({
-                    'alt_package_count_threshold': int(os.environ.get('MAX_ALTERNATE_PACKAGES', 2)),
-                    'outlier_probability_threshold': float(os.environ.get('OUTLIER_THRESHOLD',
-                                                                          0.6)),
-                    'user_persona': "1",  # TODO - remove janus hardcoded value
-                })
             input_task_for_insights_recommender = [insights_payload]
 
             # Call PGM and get the response

src/rest_api.py

@@ -54,7 +54,7 @@ def _recommender(handler):
 
     input_json = request.get_json()
     external_request_id = input_json['external_request_id']
-    logger.info('%s recommender/ request with payload: %s', external_request_id, input_json)
+    logger.info('%s recommender/ request', external_request_id)
 
     try:
         check_license = request.args.get('check_license', 'false') == 'true'
@@ -96,8 +96,7 @@ def _stack_aggregator(handler):
     }
 
     external_request_id = input_json['external_request_id']
-    logger.info('%s stack_aggregator/ request with payload: %s',
-                external_request_id, input_json)
+    logger.info('%s stack_aggregator/ request', external_request_id)
 
     try:
         persist = request.args.get('persist', 'true') == 'true'

src/settings.py

@@ -2,24 +2,34 @@
 
 
 from typing import Dict
-from pydantic import BaseSettings, HttpUrl
+from pydantic import BaseSettings, HttpUrl, AnyHttpUrl
 
 
 class AggregatorSettings(BaseSettings):
     """Create Settings from env."""
 
-    snyk_package_url_format: HttpUrl = "https://snyk.io/vuln/{ecosystem}:{package}"
+    license_analysis_base_url: AnyHttpUrl = "http://f8a-license-analysis:6162"
     snyk_signin_url: HttpUrl = "https://snyk.io/login"
+    snyk_package_url_format: HttpUrl = "https://snyk.io/vuln/{ecosystem}:{package}"
     snyk_ecosystem_map: Dict[str, str] = {"pypi": "pip"}
-    disable_unknown_package_flow: bool = False
 
 
 class RecommenderSettings(BaseSettings):
     """Create Recommender Settings from env."""
 
-    chester_service_host: str = "notset"
-    pypi_service_host: str = "notset"
-    maven_service_host: str = "notset"
-    service_port: int = 6006
+    npm_insights_base_url: AnyHttpUrl = "http://f8a-npm-insights:6006"
+    pypi_insights_base_url: AnyHttpUrl = "http://f8a-pypi-insights:6006"
+    maven_insights_base_url: AnyHttpUrl = "http://f8a-hpf-insights-maven:6006"
     unknown_packages_threshold: float = 0.3
     max_companion_packages: int = 5
+
+
+class Settings(BaseSettings):
+    """General settings."""
+
+    gremlin_url: AnyHttpUrl = "http://bayesian-gremlin-http:8182"
+
+
+SETTINGS = Settings()
+RECOMMENDER_SETTINGS = RecommenderSettings()
+AGGREGATOR_SETTINGS = AggregatorSettings()

src/stack_aggregator.py

@@ -12,9 +12,9 @@
 import requests
 import copy
 from collections import defaultdict
-from src.settings import AggregatorSettings
-from src.utils import (select_latest_version, server_create_analysis, LICENSE_SCORING_URL_REST,
-                       post_http_request, GREMLIN_SERVER_URL_REST, persist_data_in_db,
+from src.settings import AGGREGATOR_SETTINGS, SETTINGS
+from src.utils import (select_latest_version, server_create_analysis,
+                       post_http_request, persist_data_in_db,
                        GREMLIN_QUERY_SIZE, format_date)
 import logging
 
@@ -27,7 +27,7 @@ def get_recommended_version(ecosystem, name, version):
             ".out('has_version').not(out('has_cve')).values('version');"\
         .format(eco=ecosystem, pkg=name)
     payload = {'gremlin': query}
-    result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+    result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
     if result:
         versions = result['result']['data']
         if len(versions) == 0:
@@ -269,7 +269,7 @@ def _extract_license_outliers(license_service_output):
 
 def perform_license_analysis(license_score_list, dependencies):
     """Pass given license_score_list to stack_license analysis and process response."""
-    license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license"
+    license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license"
 
     payload = {
         "packages": license_score_list
@@ -510,7 +510,7 @@ def get_tr_dependency_data(epv_set):
             i = 1
             # call_gremlin in batch
             payload = {'gremlin': query}
-            result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+            result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
             if result:
                 tr_epv_list['result']['data'] += result['result']['data']
             query = "epv=[];"
@@ -519,7 +519,7 @@ def get_tr_dependency_data(epv_set):
     if i > 1:
         payload = {'gremlin': query}
         time_start = time.time()
-        result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+        result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
         logger.info('elapsed_time for gremlin call: {}'.format(time.time() - time_start))
         if result:
             tr_epv_list['result']['data'] += result['result']['data']
@@ -571,15 +571,15 @@ def get_dependency_data(epv_set):
             i = 1
             # call_gremlin in batch
             payload = {'gremlin': query}
-            result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+            result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
             if result:
                 epv_list['result']['data'] += result['result']['data']
             query = "epv=[];"
         i += 1
 
     if i > 1:
         payload = {'gremlin': query}
-        result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+        result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
         if result:
             epv_list['result']['data'] += result['result']['data']
 
@@ -663,16 +663,13 @@ def execute(aggregated=None, persist=True):
                          'result': stack_data}
         # Ingestion of Unknown dependencies
         logger.info("Unknown ingestion flow process initiated.")
-        if AggregatorSettings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', unknown_dep_list)
-        else:
-            try:
-                for dep in unknown_dep_list:
-                    server_create_analysis(ecosystem, dep['name'], dep['version'], api_flow=True,
-                                           force=False, force_graph_sync=True)
-            except Exception as e:
-                logger.error('Ingestion has been failed for ' + dep['name'])
-                logger.error(e)
-                pass
+        try:
+            for dep in unknown_dep_list:
+                server_create_analysis(ecosystem, dep['name'], dep['version'], api_flow=True,
+                                       force=False, force_graph_sync=True)
+        except Exception as e:
+            logger.error('Ingestion has been failed for ' + dep['name'])
+            logger.error(e)
+            pass
 
         return persiststatus

src/utils.py

@@ -22,6 +22,8 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import sessionmaker
 
+from src.settings import SETTINGS
+
 
 logger = logging.getLogger(__name__)
 
@@ -39,13 +41,6 @@ class RequestException(Exception):
 
 
 logger = logging.getLogger(__file__)
-GREMLIN_SERVER_URL_REST = "http://{host}:{port}".format(
-    host=os.environ.get("BAYESIAN_GREMLIN_HTTP_SERVICE_HOST", "localhost"),
-    port=os.environ.get("BAYESIAN_GREMLIN_HTTP_SERVICE_PORT", "8182"))
-
-LICENSE_SCORING_URL_REST = "http://{host}:{port}".format(
-    host=os.environ.get("LICENSE_SERVICE_HOST", "localhost"),
-    port=os.environ.get("LICENSE_SERVICE_PORT", "6162"))
 
 zero_version = sv.Version("0.0.0")
 # Create Postgres Connection Session
@@ -104,7 +99,7 @@ def get_osio_user_count(ecosystem, name, version):
         'gremlin': str_gremlin
     }
 
-    json_response = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
+    json_response = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
     return json_response.get('result').get('data', ['-1'])[0]
 
 
@@ -330,7 +325,7 @@ def post_gremlin(query: str, bindings: Dict = None) -> Dict:
     if bindings:
         payload['bindings'] = bindings
     try:
-        response = get_session_retry().post(url=GREMLIN_SERVER_URL_REST, json=payload)
+        response = get_session_retry().post(url=SETTINGS.gremlin_url, json=payload)
         response.raise_for_status()
     except Exception as e:
         logger.error(traceback.format_exc())

src/v2/license_service.py

@@ -3,7 +3,8 @@
 import logging
 from typing import List, Set
 
-from src.utils import LICENSE_SCORING_URL_REST, post_http_request
+from src.utils import post_http_request
+from src.settings import AGGREGATOR_SETTINGS
 from src.v2.models import LicenseAnalysis, PackageDetails
 
 
@@ -158,7 +159,7 @@ def get_license_service_request_payload(package_details: List[PackageDetails]):
 def get_license_analysis_for_stack(
         package_details: List[PackageDetails]) -> LicenseAnalysis:  # pylint:disable=R0914
     """Create LicenseAnalysis from license server."""
-    license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license"
+    license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license"
 
     # form payload for license service request
     payload = {