diff --git a/openshift/template.yaml b/openshift/template.yaml index 418b0052..b2f525a6 100644 --- a/openshift/template.yaml +++ b/openshift/template.yaml @@ -56,6 +56,8 @@ objects: value: f8a-pypi-insights - name: GOLANG_SERVICE_HOST value: f8a-golang-insights + - name: MAVEN_SERVICE_HOST + value: f8a-hpf-insights-maven - name: HPF_SERVICE_HOST value: f8a-hpf-insights - name: SERVICE_PORT diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh old mode 100644 new mode 100755 index 6b54fe70..9367bedd --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,4 +1,4 @@ #!/usr/bin/bash # Start API backbone service with time out -gunicorn --pythonpath /src/ -b 0.0.0.0:$API_BACKBONE_SERVICE_PORT -t $API_BACKBONE_SERVICE_TIMEOUT -k $CLASS_TYPE -w $NUMBER_WORKER_PROCESS rest_api:app \ No newline at end of file +gunicorn --pythonpath /src/ -b 0.0.0.0:$API_BACKBONE_SERVICE_PORT -t $API_BACKBONE_SERVICE_TIMEOUT -k $CLASS_TYPE -w $NUMBER_WORKER_PROCESS rest_api:app --reload --log-level $FLASK_LOGGING_LEVEL diff --git a/src/rest_api.py b/src/rest_api.py index a852fc90..afb433bd 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -1,9 +1,10 @@ """Implementation of the REST API for the backbone service.""" -import os -import logging import flask +import logging +import os import time + from f8a_worker.setup_celery import init_selinon from flask import Flask, request from flask_cors import CORS @@ -11,33 +12,17 @@ from src.recommender import RecommendationTask as RecommendationTaskV1 from src.stack_aggregator import StackAggregator as StackAggregatorV1 +from src.utils import push_data, total_time_elapsed from src.v2.recommender import RecommendationTask as RecommendationTaskV2 from src.v2.stack_aggregator import StackAggregator as StackAggregatorV2 -from src.utils import push_data, total_time_elapsed, get_time_delta - - -def setup_logging(flask_app): - """Perform the setup of logging (file, log level) for this application.""" - if not flask_app.debug: - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter( - '[%(asctime)s] %(levelname)s in %(pathname)s:%(lineno)d: %(message)s')) - log_level = os.environ.get('FLASK_LOGGING_LEVEL', logging.getLevelName(logging.WARNING)) - handler.setLevel(log_level) - - flask_app.logger.addHandler(handler) - flask_app.config['LOGGER_HANDLER_POLICY'] = 'never' - flask_app.logger.setLevel(logging.DEBUG) +logger = logging.getLogger(__name__) app = Flask(__name__) -setup_logging(app) CORS(app) SENTRY_DSN = os.environ.get("SENTRY_DSN", "") sentry = Sentry(app, dsn=SENTRY_DSN, logging=True, level=logging.ERROR) -logger = logging.getLogger(__name__) - init_selinon() @@ -68,29 +53,25 @@ def _recommender(handler): } input_json = request.get_json() - if input_json and 'external_request_id' in input_json and input_json['external_request_id']: - external_request_id = input_json['external_request_id'] - logger.info('%s recommender/ request with payload: %s', external_request_id, input_json) - - try: - check_license = request.args.get('check_license', 'false') == 'true' - persist = request.args.get('persist', 'true') == 'true' - r = handler.execute(input_json, persist=persist, - check_license=check_license) - except Exception as e: - r = { - 'recommendation': 'unexpected error', - 'external_request_id': input_json.get('external_request_id'), - 'message': '%s' % e - } - metrics_payload['status_code'] = 400 - logger.error('%s failed %s', external_request_id, r) + external_request_id = input_json['external_request_id'] + logger.info('%s recommender/ request with payload: %s', external_request_id, input_json) try: - metrics_payload['value'] = get_time_delta(audit_data=r['result']['_audit']) + check_license = request.args.get('check_license', 'false') == 'true' + persist = request.args.get('persist', 'true') == 'true' + r = handler.execute(input_json, persist=persist, + check_license=check_license) + except Exception as e: + r = { + 'recommendation': 'unexpected error', + 'external_request_id': input_json.get('external_request_id'), + 'message': '%s' % e + } + metrics_payload['status_code'] = 400 + metrics_payload['value'] = 0 push_data(metrics_payload) - except KeyError: - pass + logger.error('%s failed %s', external_request_id, r) + raise e logger.info('%s took %0.2f seconds for _recommender', external_request_id, time.time() - recommender_started_at) @@ -114,38 +95,31 @@ def _stack_aggregator(handler): 'status_code': 200 } - if input_json and 'external_request_id' in input_json \ - and input_json['external_request_id']: - external_request_id = input_json['external_request_id'] - logger.info('%s stack_aggregator/ request with payload: %s', - external_request_id, input_json) - - try: - persist = request.args.get('persist', 'true') == 'true' - s = handler.execute(input_json, persist=persist) - if s is not None and s.get('result') and s.get('result').get('_audit'): - # Creating and Pushing Total Metrics Data to Accumulator - metrics_payload['value'] = total_time_elapsed( - sa_audit_data=s['result']['_audit'], - external_request_id=input_json['external_request_id']) - push_data(metrics_payload) - - except Exception as e: - s = { - 'stack_aggregator': 'unexpected error', - 'external_request_id': input_json.get('external_request_id'), - 'message': '%s' % e - } - metrics_payload['status_code'] = 400 - logger.error('%s failed %s', external_request_id, s) - - try: - # Pushing Individual Metrics Data to Accumulator - metrics_payload['value'] = get_time_delta(audit_data=s['result']['_audit']) - metrics_payload['endpoint'] = request.endpoint + external_request_id = input_json['external_request_id'] + logger.info('%s stack_aggregator/ request with payload: %s', + external_request_id, input_json) + + try: + persist = request.args.get('persist', 'true') == 'true' + s = handler.execute(input_json, persist=persist) + if s.get('result', {}).get('_audit'): + # Creating and Pushing Total Metrics Data to Accumulator + metrics_payload['value'] = total_time_elapsed( + sa_audit_data=s['result']['_audit'], + external_request_id=input_json['external_request_id']) push_data(metrics_payload) - except KeyError: - pass + except Exception as e: + s = { + 'stack_aggregator': 'unexpected error', + 'external_request_id': input_json.get('external_request_id'), + 'message': '%s' % e + } + metrics_payload['status_code'] = 400 + # Pushing Individual Metrics Data to Accumulator + metrics_payload['value'] = 0 + push_data(metrics_payload) + logger.error('%s failed %s', external_request_id, s) + raise e logger.info('%s took %0.2f seconds for _stack_aggregators', external_request_id, time.time() - stack_aggregator_started_at) diff --git a/src/settings.py b/src/settings.py index 5c300b8b..9f6e60c7 100644 --- a/src/settings.py +++ b/src/settings.py @@ -5,11 +5,21 @@ from pydantic import BaseSettings, HttpUrl -# (fixme) Move all settings to read from here. -class Settings(BaseSettings): +class AggregatorSettings(BaseSettings): """Create Settings from env.""" - snyk_package_url_format: HttpUrl = 'https://snyk.io/vuln/{ecosystem}:{package}' - snyk_signin_url: HttpUrl = 'https://snyk.io/login' + snyk_package_url_format: HttpUrl = "https://snyk.io/vuln/{ecosystem}:{package}" + snyk_signin_url: HttpUrl = "https://snyk.io/login" snyk_ecosystem_map: Dict[str, str] = {"pypi": "pip"} disable_unknown_package_flow: bool = False + + +class RecommenderSettings(BaseSettings): + """Create Recommender Settings from env.""" + + chester_service_host: str = "notset" + pypi_service_host: str = "notset" + maven_service_host: str = "notset" + service_port: int = 6006 + unknown_packages_threshold: float = 0.3 + max_companion_packages: int = 5 diff --git a/src/stack_aggregator.py b/src/stack_aggregator.py index 219adab0..f5aaf54c 100644 --- a/src/stack_aggregator.py +++ b/src/stack_aggregator.py @@ -12,7 +12,7 @@ import requests import copy from collections import defaultdict -from src.settings import Settings +from src.settings import AggregatorSettings from src.utils import (select_latest_version, server_create_analysis, LICENSE_SCORING_URL_REST, post_http_request, GREMLIN_SERVER_URL_REST, persist_data_in_db, GREMLIN_QUERY_SIZE, format_date) @@ -663,7 +663,7 @@ def execute(aggregated=None, persist=True): 'result': stack_data} # Ingestion of Unknown dependencies logger.info("Unknown ingestion flow process initiated.") - if Settings().disable_unknown_package_flow: + if AggregatorSettings().disable_unknown_package_flow: logger.warning('Skipping unknown flow %s', unknown_dep_list) else: try: diff --git a/src/utils.py b/src/utils.py index 3c9cb2df..608fb79b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -324,21 +324,21 @@ def post_http_request(url: str, payload: Dict): def post_gremlin(query: str, bindings: Dict = None) -> Dict: """Post the given query and bindings to gremlin endpoint.""" + payload = { + 'gremlin': query, + } + if bindings: + payload['bindings'] = bindings try: - payload = { - 'gremlin': query, - } - if bindings: - payload['bindings'] = bindings response = get_session_retry().post(url=GREMLIN_SERVER_URL_REST, json=payload) response.raise_for_status() - return response.json() except Exception as e: logger.error(traceback.format_exc()) - logger.error( - "HTTP error {code}. Error retrieving data for {query}.".format( - code=response.status_code, query=payload)) + logger.error("HTTP error %d. Error retrieving data for %s.", + response.status_code, payload) raise GremlinExeception from e + else: + return response.json() def get_response_data(json_response, data_default): diff --git a/src/v2/models.py b/src/v2/models.py index 02c9c2a2..b36d20eb 100644 --- a/src/v2/models.py +++ b/src/v2/models.py @@ -120,7 +120,6 @@ class UsedByItem(BaseModel): # noqa: D101 class GitHubDetails(BaseModel): # noqa: D101 watchers: Optional[str] = None - first_release_date: Optional[str] = None total_releases: Optional[str] = None issues: Optional[Dict[str, Any]] = None pull_requests: Optional[Dict[str, Any]] = None @@ -222,7 +221,7 @@ class StackRecommendationResult(BaseModel): # noqa: D101 registration_status: str recommendation_status: 'RecommendationStatus' = 'success' companion: List['RecommendedPackageData'] - usage_outliers: List[Dict[str, Any]] + usage_outliers: List[Dict[str, Any]] = [] class RecommenderRequest(StackAggregatorRequest): # noqa: D101 @@ -230,5 +229,6 @@ class RecommenderRequest(StackAggregatorRequest): # noqa: D101 Package.update_forward_refs() +PackageDetails.update_forward_refs() PackageDataWithVulnerabilities.update_forward_refs() RecommendedPackageData.update_forward_refs() diff --git a/src/v2/openapi.yaml b/src/v2/openapi.yaml index d09925d7..997cc662 100644 --- a/src/v2/openapi.yaml +++ b/src/v2/openapi.yaml @@ -360,8 +360,6 @@ components: watchers: type: string example: "100K" - first_release_date: - type: string total_releases: type: string example: "12" @@ -487,7 +485,6 @@ components: - recommendation_status - companion - manifest_file_path - - usage_outliers properties: _audit: $ref: '#/components/schemas/Audit' @@ -513,11 +510,6 @@ components: manifest_file_path: type: string example: pom.xml - usage_outliers: - type: array - items: - type: object - properties: {} description: Application stack recommendations Audit: diff --git a/src/v2/recommender.py b/src/v2/recommender.py index 7cb0a68a..328f91a6 100644 --- a/src/v2/recommender.py +++ b/src/v2/recommender.py @@ -4,474 +4,207 @@ sourcing. """ -import json import datetime -import requests -import os -import time import logging -from collections import defaultdict - -from src.utils import (create_package_dict, get_session_retry, select_latest_version, - LICENSE_SCORING_URL_REST, convert_version_to_proper_semantic, - get_response_data, version_info_tuple, persist_data_in_db, - post_gremlin) -from src.v2.models import RecommenderRequest, StackRecommendationResult -from src.v2.stack_aggregator import extract_user_stack_package_licenses -from src.v2.normalized_packages import NormalizedPackages - - -logger = logging.getLogger(__name__) - - -class GraphDB: - """Graph database interface.""" +import time +from typing import List, Dict - @staticmethod - def get_version_information(input_list, ecosystem): - """Fetch the version information for each of the packages. +from pydantic import BaseModel - Also remove EPVs with CVEs and ones not present in Graph - """ - str_query = "data=[]; " - for package in input_list: - str_query += "pkg = g.V().has('ecosystem', '{eco}').has('name', '{pkg}'); " \ - "lnv = []; pkg.clone().values('latest_non_cve_version', " \ - "'latest_version').fill(lnv); pkg.clone().as('package').V()." \ - "has('pecosystem', '{eco}').has('pname', '{pkg}')." \ - "has('version', within(lnv)).as('version')." \ - "select('package', 'version').by(valueMap()).fill(data);".format( - eco=ecosystem, - pkg=package) - str_query += "data" - # Query Gremlin with packages list to get their version information - gremlin_response = post_gremlin(str_query) - if gremlin_response is None: - return [] - response = get_response_data(gremlin_response, [{0: 0}]) - return response +from src.utils import get_session_retry, persist_data_in_db, post_gremlin +from src.v2.stack_aggregator import get_github_details, get_snyk_package_link +from src.v2.normalized_packages import NormalizedPackages +from src.settings import RecommenderSettings - @staticmethod - def add_version_to_filtered_list(epv, key, val, semversion_tuple, input_stack_tuple, - pkg_dict, new_dict, filtered_comp_list): - """Add versions to filtered list.""" - name = epv.get('package', {}).get('name', [''])[0] - version = epv.get('version', {}).get('version', [''])[0] - try: - if semversion_tuple >= input_stack_tuple: - pkg_dict[name][key] = {"version": version, key: val} - new_dict[name][key] = epv.get('version') - new_dict[name]['package'] = epv.get('package') - filtered_comp_list.append(name) +from src.v2.models import ( + RecommenderRequest, + StackRecommendationResult, + Ecosystem, + RecommendedPackageData, +) - except ValueError: - logger.exception("Unexpected ValueError while filtering latest version!") - pass - return pkg_dict, new_dict, filtered_comp_list +logger = logging.getLogger(__name__) - @staticmethod - def prepare_final_filtered_list(new_dict): - """Prepare filtered list of versions.""" - new_list = [] - for package, contents in new_dict.items(): - if 'latest_version' in contents: - new_list.append({"package": contents['package'], - "version": contents['latest_version']}) - elif 'deps_count' in contents: - new_list.append({"package": contents['package'], - "version": contents['deps_count']}) - elif 'gh_release_date' in contents: - new_list.append({"package": contents['package'], - "version": contents['gh_release_date']}) - - return new_list - @staticmethod - def filter_versions(epv_list, input_stack, external_request_id=None, rec_type=None): - """Filter the EPVs according to following rules. - - First filter fetches only EPVs that - 1. has No CVEs - 2. are Present in Graph - Apply additional filter based on following. That is sorted based on - 3. Latest Version - 4. Dependents Count in Github Manifest Data - 5. Github Release Date - """ - logger.debug('%s filtering %s', external_request_id, rec_type) - - pkg_dict = defaultdict(dict) - new_dict = defaultdict(dict) - filtered_comp_list = [] - - for epv in epv_list: - name = epv.get('package', {}).get('name', [''])[0] - version = epv.get('version', {}).get('version', [''])[0] - libio_latest_version = epv.get('package').get('libio_latest_version', [''])[0] - latest_version = epv.get('package').get('latest_version', [''])[0] - - # Convert version to a proper semantic case - semversion_tuple = version_info_tuple( - convert_version_to_proper_semantic(version, name)) - input_stack_tuple = version_info_tuple( - convert_version_to_proper_semantic(input_stack.get(name, ''), name)) - - if name and version: - # Select highest version based on input or graph as latest version - latest_version = select_latest_version( - version, libio_latest_version, latest_version, name - ) - - if latest_version and latest_version == version: - pkg_dict, new_dict, filtered_comp_list = GraphDB.add_version_to_filtered_list( - epv=epv, key='latest_version', val=latest_version, pkg_dict=pkg_dict, - new_dict=new_dict, filtered_comp_list=filtered_comp_list, - semversion_tuple=semversion_tuple, input_stack_tuple=input_stack_tuple) - - # Select Version based on highest dependents count (usage) - deps_count = epv.get('version').get('dependents_count', [-1])[0] - if deps_count > 0: - if 'deps_count' not in pkg_dict[name] or \ - deps_count > pkg_dict[name].get('deps_count').get('deps_count', 0): - pkg_dict, new_dict, filtered_comp_list = \ - GraphDB.add_version_to_filtered_list( - epv=epv, key='deps_count', val=deps_count, pkg_dict=pkg_dict, - new_dict=new_dict, filtered_comp_list=filtered_comp_list, - semversion_tuple=semversion_tuple, - input_stack_tuple=input_stack_tuple) - - # Select Version with the most recent github release date - gh_release_date = epv.get('version').get('gh_release_date', [0.0])[0] - if gh_release_date > 0.0: - if 'gh_release_date' not in pkg_dict[name] or \ - gh_release_date > pkg_dict[name]['gh_release_date'].\ - get('gh_release_date', 0.0): - pkg_dict, new_dict, filtered_comp_list = \ - GraphDB.add_version_to_filtered_list( - epv=epv, key='gh_release_date', val=gh_release_date, - pkg_dict=pkg_dict, new_dict=new_dict, - filtered_comp_list=filtered_comp_list, - semversion_tuple=semversion_tuple, - input_stack_tuple=input_stack_tuple) - - logger.debug('%s new dict %s', external_request_id, json.dumps(new_dict)) - logger.debug( - '%s filtered comp list %s', external_request_id, json.dumps(filtered_comp_list)) - - new_list = GraphDB.prepare_final_filtered_list(new_dict) - return new_list, filtered_comp_list +class InsightsRequest(BaseModel): + """InsightsRequest payload model.""" - @staticmethod - def get_topics_for_comp(comp_list, pgm_list): - """Get topics from pgm and associate with filtered versions from Graph.""" - for epv in comp_list: - name = epv.get('package', {}).get('name', [''])[0] - if name: - for pgm_epv in pgm_list: - if name == pgm_epv.get('package_name', ''): - epv['package']['topic_list'] = pgm_epv.get('topic_list', []) - epv['package']['cooccurrence_probability'] = pgm_epv.get( - 'cooccurrence_probability', 0) - epv['package']['cooccurrence_count'] = pgm_epv.get( - 'cooccurrence_count', 0) + ecosystem: Ecosystem + transitive_stack: List[str] = [] + package_list: List[str] + unknown_packages_ratio_threshold: float = ( + RecommenderSettings().unknown_packages_threshold + ) + comp_package_count_threshold: int = RecommenderSettings().max_companion_packages - return comp_list +class InsightsCallException(Exception): + """Exception related to insight service call failures.""" -class License: - """License Analytics Class.""" - @staticmethod - def invoke_license_analysis_service(user_stack_packages, comp_packages): - """Pass given args to stack_license analysis.""" - license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license" +class InsightsWithEmptyPackageException(Exception): + """Exception for empty request packages.""" - payload = { - "packages": user_stack_packages, - "companion_packages": comp_packages - } - json_response = {} - try: - # Call License service to get license data - lic_response = get_session_retry().post(license_url, data=json.dumps(payload)) - if lic_response.status_code != 200: - lic_response.raise_for_status() # raise exception for bad http-status codes - json_response = lic_response.json() - except requests.exceptions.RequestException: - logger.exception("Unexpected error happened while invoking license analysis!") - pass +def _prepare_insights_url(host: str) -> str: + assert host + url = "http://{host}:{port}".format( + host=host, port=RecommenderSettings().service_port + ) + endpoint = "{url}/api/v1/companion_recommendation".format(url=url) + return endpoint - return json_response - @staticmethod - def apply_license_filter(user_stack_components, epv_list_com): - """Get License Analysis and filter out License Conflict EPVs.""" - license_score_list_com = [] - conflict_packages_com = [] - list_pkg_names_com = [] - - for epv in epv_list_com: - license_scoring_input = { - 'package': epv.get('package', {}).get('name', [''])[0], - 'version': epv.get('version', {}).get('version', [''])[0], - 'licenses': epv.get('version', {}).get('declared_licenses', []) - } - license_score_list_com.append(license_scoring_input) - - # Call license scoring to find license filters - la_output = License.invoke_license_analysis_service(user_stack_components, - license_score_list_com) - - if la_output.get('status') == 'Successful' and la_output.get('license_filter') is not None: - license_filter = la_output.get('license_filter', {}) - conflict_packages_com = license_filter.get('companion_packages', {}) \ - .get('conflict_packages', []) - - for epv in epv_list_com[:]: - name = epv.get('package', {}).get('name', [''])[0] - if name in conflict_packages_com: - list_pkg_names_com.append(name) - epv_list_com.remove(epv) - - output = { - 'filtered_comp_packages_graph': epv_list_com, - 'filtered_list_pkg_names_com': list_pkg_names_com - } - logger.info("License Filter output: {}".format(json.dumps(output))) - - return output - - @staticmethod - def perform_license_analysis( - packages, filtered_companion_packages, - filtered_comp_packages_graph, external_request_id): - """Apply License Filters and log the messages.""" - list_user_stack_comp = extract_user_stack_package_licenses(packages) - license_filter_output = License.apply_license_filter( - list_user_stack_comp, - filtered_comp_packages_graph) - - lic_filtered_comp_graph = license_filter_output['filtered_comp_packages_graph'] - lic_filtered_list_com = license_filter_output['filtered_list_pkg_names_com'] - - if len(lic_filtered_list_com) > 0: - s = set(filtered_companion_packages).difference(set(lic_filtered_list_com)) - logger.info( - '%s Companion Packages filtered (licenses) %s', external_request_id, s) - - return lic_filtered_comp_graph - - -def set_valid_cooccurrence_probability(package_list=[]): - """Return a list of companion components with valid co-occurrence probability. - - :param package_list: - :return: list of valid companion components - """ - new_package_list = [] - for package in package_list: - if str(package['cooccurrence_probability']) == 'nan': - logger.error("Found an invalid cooccurrence probability for %s" % package['name']) - package['cooccurrence_probability'] = float(100.0) - new_package_list.append(package) - return new_package_list +ECOSYSTEM_TO_INSIGHTS_URL = { + "pypi": _prepare_insights_url(RecommenderSettings().pypi_service_host), + "npm": _prepare_insights_url(RecommenderSettings().chester_service_host), + "maven": _prepare_insights_url(RecommenderSettings().maven_service_host), +} class RecommendationTask: """Recommendation task.""" - _analysis_name = 'recommendation_v2' - description = 'Get Recommendation' - kronos_ecosystems = ['maven'] - chester_ecosystems = ['npm'] - hpf_ecosystems = ['maven'] - pypi_ecosystems = ['pypi'] - golang_ecosystem = ['golang'] - - @staticmethod - def get_insights_url(payload): - """Get the insights url based on the ecosystem.""" - if payload and 'ecosystem' in payload[0]: - if payload[0]['ecosystem'] in RecommendationTask.chester_ecosystems: - INSIGHTS_SERVICE_HOST = os.getenv("CHESTER_SERVICE_HOST") - elif payload[0]['ecosystem'] in RecommendationTask.pypi_ecosystems: - INSIGHTS_SERVICE_HOST = os.getenv("PYPI_SERVICE_HOST") - elif payload[0]['ecosystem'] in RecommendationTask.golang_ecosystem: - INSIGHTS_SERVICE_HOST = os.environ.get("GOLANG_SERVICE_HOST") - else: - INSIGHTS_SERVICE_HOST = os.getenv("HPF_SERVICE_HOST") + "-" + payload[0][ - 'ecosystem'] - - INSIGHTS_URL_REST = "http://{host}:{port}".format(host=INSIGHTS_SERVICE_HOST, - port=os.getenv("SERVICE_PORT")) - - insights_url = INSIGHTS_URL_REST + "/api/v1/companion_recommendation" - - return insights_url - - else: - logger.error('Payload information not passed in the call, Quitting! inights ' - 'recommender\'s call') - @staticmethod - def call_insights_recommender(payload): + def _call(payload: InsightsRequest): """Call the PGM model. Calls the PGM model with the normalized manifest information to get the relevant packages. """ + insights_url = ECOSYSTEM_TO_INSIGHTS_URL.get(payload.ecosystem, None) + assert insights_url try: - # TODO remove hardcodedness for payloads with multiple ecosystems - - insights_url = RecommendationTask.get_insights_url(payload) - response = get_session_retry().post(insights_url, json=payload) - - if response.status_code != 200: - logger.error("HTTP error {}. Error retrieving insights data.".format( - response.status_code)) - return None - else: - json_response = response.json() - return json_response - + started_at = time.time() + response = get_session_retry().post(insights_url, json=[payload.dict()]) + response.raise_for_status() except Exception as e: - logger.error("Failed retrieving insights data.") - logger.error("%s" % e) - return None - - def execute(self, arguments=None, persist=True, check_license=False): + raise InsightsCallException() from e + else: + json_response = response.json() + logger.info( + "Recommendation [%s] req.pkgs [%d] elapsed time [%0.2f] sec", + payload.ecosystem.value, + len(payload.package_list), + time.time() - started_at, + ) + return json_response + + def _get_insights_response(self, normalized_packages: NormalizedPackages): + package_list = list( + map(lambda epv: epv.name, normalized_packages.direct_dependencies) + ) + if not package_list: + raise InsightsWithEmptyPackageException("Request package list is empty") + + insights_payload = InsightsRequest( + ecosystem=normalized_packages.ecosystem, + transitive_stack=list( + map(lambda epv: epv.name, normalized_packages.transitive_dependencies) + ), + package_list=package_list, + ) + # Call PGM and get the response + return self._call(insights_payload) + + def _get_recommended_package_details( + self, insights_response + ) -> List[RecommendedPackageData]: + companion_packages = insights_response.get("companion_packages", []) + package_to_stats_map = dict( + map( + lambda x: (x.get("package_name"), x), + companion_packages, + ) + ) + packages = list(package_to_stats_map.keys()) + ecosystem = insights_response["ecosystem"] + query = ( + """g.V().has('ecosystem', ecosystem).has('name', within(name)).valueMap()""" + ) + started_at = time.time() + result = post_gremlin( + query=query, bindings={"ecosystem": ecosystem, "name": packages} + ) + logger.info( + "graph req.pkgs [%d] elapsed time [%0.2f] sec", + len(packages), + time.time() - started_at, + ) + + def extract_version(data): + # all versions are not vulnerable if latest_non_cve_version doesn't exist. + # all versions are vulnerable if latest_non_cve_version is empty. + recommended_version = data.get( + "latest_non_cve_version", data.get("latest_version", [""]) + ) + version = recommended_version[0] if len(recommended_version) else "" + return version + + def has_valid_version(data): + return extract_version(data) != "" + + def get_recommendation_statistics(package_name: str) -> Dict[str, str]: + # below dict has cooccurrence_probability, cooccurrence_count, topic_list + return package_to_stats_map[package_name] + + def map_to_recommendation_package_data(data): + name = data.get("name", [""])[0] + version = extract_version(data) + return RecommendedPackageData( + name=name, + version=version, + github=get_github_details(data), + licenses=data.get("declared_licenses", []), + ecosystem=ecosystem, + url=get_snyk_package_link(ecosystem, name), + latest_version=data.get("latest_version", [""])[0], + # join stats from insight + **get_recommendation_statistics(name), + ) + + valid_packages = filter(has_valid_version, result["result"]["data"]) + return list(map(map_to_recommendation_package_data, valid_packages)) + + def execute(self, arguments=None, persist=True, check_license=False): # noqa: F841 """Execute task.""" started_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") request = RecommenderRequest(**arguments) - external_request_id = request.external_request_id + if request.ecosystem == "golang": + logging.warning("Recommendation is yet to be implemented for golang") + return {} normalized_packages = NormalizedPackages(request.packages, request.ecosystem) + insights_response = self._get_insights_response(normalized_packages) - recommendation = { - 'companion': [], - 'usage_outliers': [], - } - package_list = [epv.name for epv in normalized_packages.direct_dependencies] - if package_list: - insights_payload = { - 'ecosystem': request.ecosystem, - 'transitive_stack': [ - epv.name for epv in normalized_packages.transitive_dependencies], - 'unknown_packages_ratio_threshold': - float(os.environ.get('UNKNOWN_PACKAGES_THRESHOLD', 0.3)), - 'package_list': package_list, - 'comp_package_count_threshold': int(os.environ.get( - 'MAX_COMPANION_PACKAGES', 5)) - } - if request.ecosystem in self.kronos_ecosystems: - insights_payload.update({ - 'outlier_probability_threshold': float(os.environ.get('OUTLIER_THRESHOLD', - 0.6)), - 'user_persona': "1", # TODO - remove janus hardcoded value - }) - input_task_for_insights_recommender = [insights_payload] - - # Call PGM and get the response - insights_response = [] - start = time.time() - if request.ecosystem != 'golang': - # No Companion Rec. for Golang. - insights_response = self.call_insights_recommender( - input_task_for_insights_recommender) - - logger.info('%s took %0.2f secs for call_insights_recommender()', - external_request_id, time.time() - start) - - # From PGM response process companion and alternate packages and - # then get Data from Graph - # TODO - implement multiple manifest file support for below loop - - if insights_response is None: - return { - 'recommendation': 'pgm_error', - 'external_request_id': external_request_id, - 'message': 'PGM Fetching error' - } - - for insights_result in insights_response: - companion_packages = [] - ecosystem = insights_result['ecosystem'] - - # Get usage based outliers - recommendation['usage_outliers'] = \ - insights_result.get('outlier_package_list', []) - - # Append Topics for User Stack - recommendation['input_stack_topics'] = insights_result.get( - 'package_to_topic_dict', {}) - # Add missing packages unknown to PGM - recommendation['missing_packages_pgm'] = insights_result.get( - 'missing_packages', []) - for pkg in insights_result['companion_packages']: - companion_packages.append(pkg['package_name']) - - # Get Companion Packages from Graph - graph_request_started_at = time.time() - comp_packages_graph = GraphDB().get_version_information(companion_packages, - ecosystem) - logger.info( - '%s took %0.2f secs for GraphDB().get_version_information()', - external_request_id, time.time() - graph_request_started_at) - - # Apply Version Filters - input_stack = { - epv.name: epv.version for epv in normalized_packages.direct_dependencies} - filtered_comp_packages_graph, filtered_list = GraphDB().filter_versions( - comp_packages_graph, input_stack, external_request_id, rec_type="COMPANION") - - filtered_companion_packages = \ - set(companion_packages).difference(set(filtered_list)) - logger.info('%s Fitered companion packages %s', - external_request_id, filtered_companion_packages) - - if check_license: - # Apply License Filters - license_request_started_at = time.time() - lic_filtered_comp_graph = \ - License.perform_license_analysis( - packages=normalized_packages, - filtered_comp_packages_graph=filtered_comp_packages_graph, - filtered_companion_packages=filtered_companion_packages, - external_request_id=external_request_id - ) - logger.info( - '%s took %0.2f secs for License.perform_license_analysis()', - external_request_id, time.time() - license_request_started_at) - else: - lic_filtered_comp_graph = filtered_comp_packages_graph - - # Get Topics Added to Filtered Packages - topics_comp_packages_graph = GraphDB(). \ - get_topics_for_comp(lic_filtered_comp_graph, - insights_result.get('companion_packages', [])) - - # Create Companion Block - comp_packages = create_package_dict(topics_comp_packages_graph) - final_comp_packages = \ - set_valid_cooccurrence_probability(comp_packages) - - recommendation['companion'] = final_comp_packages + result = StackRecommendationResult( + **arguments, + companion=self._get_recommended_package_details(insights_response[0]), + ) + recommendation = result.dict() ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") - audit = {'started_at': started_at, 'ended_at': ended_at, 'version': 'v2'} - - recommendation = StackRecommendationResult(**recommendation, **request.dict()).dict() - recommendation['_audit'] = audit + recommendation["_audit"] = { + "started_at": started_at, + "ended_at": ended_at, + "version": "v2", + } + external_request_id = request.external_request_id if persist: - persist_data_in_db(external_request_id=external_request_id, - task_result=recommendation, worker='recommendation_v2', - started_at=started_at, ended_at=ended_at) + persist_data_in_db( + external_request_id=external_request_id, + task_result=recommendation, + worker="recommendation_v2", + started_at=started_at, + ended_at=ended_at, + ) logger.info( - '%s Recommendation process completed, result persisted into RDS.', - external_request_id) - - return {'recommendation': 'success', - 'external_request_id': external_request_id, - 'result': recommendation} + "%s Recommendation process completed, result persisted into RDS.", + external_request_id, + ) + return { + "recommendation": "success", + "external_request_id": external_request_id, + "result": recommendation, + } diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py index e74c653b..9d182cf7 100644 --- a/src/v2/stack_aggregator.py +++ b/src/v2/stack_aggregator.py @@ -14,7 +14,7 @@ from typing import Dict, List, Tuple, Set from f8a_utils.gh_utils import GithubUtils -from src.settings import Settings +from src.settings import AggregatorSettings from src.utils import (select_latest_version, server_create_analysis, persist_data_in_db, post_gremlin, GREMLIN_QUERY_SIZE, format_date) @@ -56,7 +56,7 @@ def _get_vulnerability_fields(vuln_node: Dict[str, str]): } -def _get_github_details(package_node) -> GitHubDetails: +def get_github_details(package_node) -> GitHubDetails: """Get fields associated with Github statistics of a package node.""" date = format_date(package_node.get("gh_refreshed_on", ["N/A"])[0]) github_details = { @@ -65,9 +65,9 @@ def _get_github_details(package_node) -> GitHubDetails: "dependent_repos": package_node.get("libio_dependents_repos", [-1])[0], "total_releases": package_node.get("libio_total_releases", [-1])[0], "latest_release_duration": - str(datetime.datetime.fromtimestamp(package_node.get( + str(datetime.datetime.utcfromtimestamp(package_node.get( "libio_latest_release", [1496302486.0])[0])), - "first_release_date": "Apr 16, 2010", + "watchers": package_node.get("gh_subscribers_count", [-1])[0], "issues": { "month": { "opened": package_node.get("gh_issues_last_month_opened", [-1])[0], @@ -120,6 +120,44 @@ def extract_user_stack_package_licenses(packages: NormalizedPackages): return get_license_service_request_payload(normalized_package_details) +def _get_vulnerabilities(vulnerability_nodes): + """Get list of vulnerabilities associated with a package.""" + public_vulns = [] + private_vulns = [] + for vuln in vulnerability_nodes: + if _is_private_vulnerability(vuln): + private_vulns.append(VulnerabilityFields(**_get_vulnerability_fields(vuln))) + else: + public_vulns.append(VulnerabilityFields(**_get_vulnerability_fields(vuln))) + return public_vulns, private_vulns + + +def get_package_details(component: Dict) -> PackageDataWithVulnerabilities: + """Extract package details from given graph response.""" + pkg_node = component.get("package", {}) + version_node = component.get("version", {}) + ecosystem, pkg = _get_pkg_from_graph_version_node(version_node) + github_details = get_github_details(pkg_node) + public_vulns, private_vulns = _get_vulnerabilities(component.get("vuln", {})) + recommended_latest_version = pkg_node.get("latest_non_cve_version", [""])[0] + licenses = version_node.get("declared_licenses", []) + + latest_version = select_latest_version( + pkg.version, + pkg_node.get("libio_latest_version", [""])[0], + pkg_node.get("latest_version", [""])[0], + pkg.name + ) + return pkg, PackageDataWithVulnerabilities(**pkg.dict(), ecosystem=ecosystem, + latest_version=latest_version, + github=github_details, licenses=licenses, + # (fixme) this is incorrect + url=get_snyk_package_link(ecosystem, pkg.name), + private_vulnerabilities=private_vulns, + public_vulnerabilities=public_vulns, + recommended_version=recommended_latest_version) + + def _get_packages_in_batch(dependencies: Tuple[Package], size: int) -> Tuple[Package]: """Take Package Tuple and slices it according to size.""" for i in range(0, len(dependencies), size): @@ -131,10 +169,11 @@ def _has_vulnerability(pkg: PackageDetails) -> bool: # (fixme) link to snyk package should be identified during ingestion. -def _get_snyk_package_link(ecosystem: str, package: str) -> str: - ecosystem = Settings().snyk_ecosystem_map.get(ecosystem, ecosystem) - return Settings().snyk_package_url_format.format(ecosystem=ecosystem, - package=quote(package, safe='')) +def get_snyk_package_link(ecosystem: str, package: str) -> str: + """Prepare snyk package link based on ecosystem and package name.""" + ecosystem = AggregatorSettings().snyk_ecosystem_map.get(ecosystem, ecosystem) + return AggregatorSettings().snyk_package_url_format.format(ecosystem=ecosystem, + package=quote(package, safe='')) class Aggregator: @@ -154,48 +193,11 @@ def get_package_details_from_graph(self) -> Dict[Package, PackageDetails]: graph_response = self._get_package_details_with_vulnerabilities() package_details: List[Tuple[Package, PackageDetails]] = [] for pkg in graph_response: - package_details.append(self._get_package_details(pkg)) + package_details.append(get_package_details(pkg)) # covert list of (pkg, package_details) into map return dict(package_details) - def _get_vulnerabilities(self, vulnerability_nodes): - """Get list of vulnerabilities associated with a package.""" - public_vulns = [] - private_vulns = [] - for vuln in vulnerability_nodes: - if _is_private_vulnerability(vuln): - private_vulns.append(VulnerabilityFields(**_get_vulnerability_fields(vuln))) - else: - public_vulns.append(VulnerabilityFields(**_get_vulnerability_fields(vuln))) - return public_vulns, private_vulns - - def _get_package_details(self, component): - """Extract package details from given graph response.""" - pkg_node = component.get("package", {}) - version_node = component.get("version", {}) - ecosystem, pkg = _get_pkg_from_graph_version_node(version_node) - github_details = _get_github_details(pkg_node) - public_vulns, private_vulns = self._get_vulnerabilities(component.get("vuln", {})) - recommended_latest_version = pkg_node.get("latest_non_cve_version", [""])[0] - licenses = version_node.get("declared_licenses", []) - - latest_version = select_latest_version( - pkg.version, - pkg_node.get("libio_latest_version", [""])[0], - pkg_node.get("latest_version", [""])[0], - pkg.name - ) - return pkg, PackageDataWithVulnerabilities(**pkg.dict(), ecosystem=ecosystem, - latest_version=latest_version, - github=github_details, licenses=licenses, - # (fixme) this is incorrect - url=_get_snyk_package_link(ecosystem, - pkg.name), - private_vulnerabilities=private_vulns, - public_vulnerabilities=public_vulns, - recommended_version=recommended_latest_version) - def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]: """Get package data from graph along with vulnerability.""" time_start = time.time() @@ -300,11 +302,11 @@ def get_result(self) -> StackAggregatorResult: analyzed_dependencies=package_details, unknown_dependencies=unknown_dependencies, license_analysis=license_analysis, - registration_link=Settings().snyk_signin_url) + registration_link=AggregatorSettings().snyk_signin_url) def initiate_unknown_package_ingestion(self): """Ingestion of Unknown dependencies.""" - if Settings().disable_unknown_package_flow: + if AggregatorSettings().disable_unknown_package_flow: logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages()) return @@ -381,7 +383,7 @@ def __init__(self, request: StackAggregatorRequest = None, def initiate_unknown_package_ingestion(self): """Ingestion of Unknown dependencies.""" - if Settings().disable_unknown_package_flow: + if AggregatorSettings().disable_unknown_package_flow: logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages()) return logger.error('Ingestion is Not active for Golang.') @@ -414,7 +416,7 @@ def get_package_details_from_graph(self) -> Dict[Package, PackageDetails]: graph_response = self._get_package_details_with_vulnerabilities() package_details: List[Tuple[Package, PackageDetails]] = [] for pkg in graph_response: - package_details.append(self._get_package_details(pkg)) + package_details.append(get_package_details(pkg)) psedo_pkgs_data = self._get_package_details_from_graph_for_pseudo_versions() for pseudo_pkg in psedo_pkgs_data: @@ -480,7 +482,7 @@ def _get_golang_package_details(self, pkg_node) -> Tuple[Package, PackageDetails ecosystem = pkg_node.get('ecosystem', [''])[0] pkg = Package(name=pkg_name, version=self._normalized_packages.version_map[pkg_name]) latest_version = pkg_node.get('latest_version', [''])[0] - public_vulns, private_vulns = self._get_vulnerabilities( + public_vulns, private_vulns = _get_vulnerabilities( self.filtered_vul.get(pkg_name, [])) recommended_latest_version = pkg_node.get("latest_non_cve_version", [""])[0] pkg_details = PackageDataWithVulnerabilities( @@ -489,7 +491,7 @@ def _get_golang_package_details(self, pkg_node) -> Tuple[Package, PackageDetails latest_version=latest_version, github={}, licenses=[], - url=_get_snyk_package_link(ecosystem, pkg_name), + url=get_snyk_package_link(ecosystem, pkg_name), private_vulnerabilities=private_vulns, public_vulnerabilities=public_vulns, recommended_version=recommended_latest_version) diff --git a/tests/v2/data/graph_package_response_for_npm_express_winston_moment.py b/tests/v2/data/graph_package_response_for_npm_express_winston_moment.py new file mode 100644 index 00000000..c7031464 --- /dev/null +++ b/tests/v2/data/graph_package_response_for_npm_express_winston_moment.py @@ -0,0 +1,135 @@ +"""Mock data from graph for npm express and winston.""" + +DATA = { + "result": { + "data": [ + { + "gh_issues_last_month_opened": [18], + "gh_prs_last_year_closed": [61], + "libio_usedby": [ + "Automattic/mongoose:14372", + "ReactTraining/react-router:27232", + "angular/angular.js:57749", + "angular/angular:31547", + "facebook/jest:14465", + "facebook/react:84669", + "postcss/autoprefixer:14331", + "reactjs/redux:36922", + "socketio/socket.io:38209", + "webpack/webpack:35429", + ], + "ecosystem": ["npm"], + "gh_subscribers_count": [1799], + "gh_contributors_count": [238], + "latest_version_last_updated": ["20201120"], + "vertex_label": ["Package"], + "libio_dependents_repos": ["893994"], + "latest_non_cve_version": ["4.17.1"], + "gh_issues_last_year_opened": [-1], + "gh_issues_last_month_closed": [18], + "gh_open_issues_count": [155], + "libio_dependents_projects": ["48988"], + "latest_version": ["4.17.1"], + "tokens": ["express"], + "package_relative_used": ["not used"], + "gh_stargazers": [51280], + "gh_forks": [8571], + "package_dependents_count": [-1], + "gh_prs_last_month_opened": [3], + "gh_issues_last_year_closed": [-1], + "last_updated": [1.6087431764215431e9], + "gh_prs_last_month_closed": [0], + "topics": ["express", "javascript", "nodejs", "server"], + "libio_total_releases": ["291"], + "gh_refreshed_on": ["2020-12-23 16:28:34"], + "gh_prs_last_year_opened": [75], + "name": ["express"], + "libio_latest_version": ["5.0.0-alpha.7"], + "libio_latest_release": [1.540609931e9], + }, + { + "gh_issues_last_month_opened": [-1], + "gh_prs_last_year_closed": [-1], + "libio_usedby": [ + "amir20/phantomjs-node:2884", + "angular/angular.js:56712", + "angular/angular:26987", + "angular/material2:10337", + "apidoc/apidoc:4685", + "cyclejs/cyclejs:7180", + "flatiron/prompt:1142", + "foreverjs/forever:9868", + "ionic-team/ionic-native:1384", + "ionic-team/ionic:30889", + ], + "ecosystem": ["npm"], + "gh_subscribers_count": [224], + "gh_contributors_count": [30], + "vertex_label": ["Package"], + "libio_dependents_repos": ["35152"], + "gh_issues_last_year_opened": [-1], + "gh_issues_last_month_closed": [-1], + "gh_open_issues_count": [236], + "libio_dependents_projects": ["6413"], + "latest_version": ["3.2.1"], + "tokens": ["winston"], + "package_relative_used": ["not used"], + "gh_stargazers": [12020], + "gh_forks": [1129], + "package_dependents_count": [-1], + "gh_prs_last_month_opened": [-1], + "gh_issues_last_year_closed": [-1], + "last_updated": [1.560302668156725e9], + "gh_prs_last_month_closed": [-1], + "libio_total_releases": ["73"], + "gh_prs_last_year_opened": [-1], + "name": ["winston"], + "libio_latest_version": ["3.0.0"], + "libio_latest_release": [1.528821086e9], + }, + { + "gh_issues_last_month_opened": [-1], + "gh_prs_last_year_closed": [-1], + "libio_usedby": [ + "DefinitelyTyped/DefinitelyTyped:12868", + "ReactTraining/react-router:25916", + "SBoudrias/Inquirer.js:5149", + "angular/angular.js:57223", + "angular/angular:29571", + "angular/zone.js:2334", + "babel/babel:23356", + "facebook/jest:13148", + "facebook/react:80053", + "webpack/webpack:33424", + ], + "ecosystem": ["npm"], + "gh_subscribers_count": [923], + "gh_contributors_count": [416], + "latest_version_last_updated": ["20201211"], + "vertex_label": ["Package"], + "libio_dependents_repos": ["238544"], + "latest_non_cve_version": [], + "gh_issues_last_year_opened": [-1], + "gh_issues_last_month_closed": [-1], + "gh_open_issues_count": [101], + "libio_dependents_projects": ["38749"], + "latest_version": ["2.29.1"], + "tokens": ["moment"], + "package_relative_used": ["not used"], + "gh_stargazers": [45252], + "gh_forks": [6824], + "package_dependents_count": [-1], + "gh_prs_last_month_opened": [-1], + "gh_issues_last_year_closed": [-1], + "last_updated": [1.6087389910452957e9], + "gh_prs_last_month_closed": [-1], + "libio_total_releases": ["71"], + "gh_refreshed_on": ["2020-12-23 14:50:07"], + "gh_prs_last_year_opened": [-1], + "name": ["moment"], + "libio_latest_version": ["2.27.0"], + "libio_latest_release": [1.592517656e9], + }, + ], + } +} diff --git a/tests/v2/data/npm_insights_response_for_request.py b/tests/v2/data/npm_insights_response_for_request.py new file mode 100644 index 00000000..ac10e2ef --- /dev/null +++ b/tests/v2/data/npm_insights_response_for_request.py @@ -0,0 +1,44 @@ +"""Mock data from npm insights.""" + +DATA = [ + { + "companion_packages": [ + { + "cooccurrence_probability": 52.50463667254613, + "package_name": "express", + "topic_list": [ + "accepts", + ], + }, + { + "cooccurrence_probability": 51.04134789351682, + "package_name": "winston", + "topic_list": [ + "async", + ], + }, + { + "cooccurrence_probability": 49.54337206255131, + "package_name": "moment", + "topic_list": [], + }, + { + "cooccurrence_probability": 49.18425744667468, + "package_name": "colors", + "topic_list": [], + }, + { + "cooccurrence_probability": 47.72579254602449, + "package_name": "commander", + "topic_list": [], + }, + ], + "ecosystem": "npm", + "missing_packages": [], + "package_to_topic_dict": { + "request": [ + "aws-sign2", + ] + }, + } +] diff --git a/tests/v2/test_recommender.py b/tests/v2/test_recommender.py deleted file mode 100644 index 0a5645cd..00000000 --- a/tests/v2/test_recommender.py +++ /dev/null @@ -1,285 +0,0 @@ -"""Tests for the recommender module.""" -from unittest import TestCase, mock -import json - - -from src.v2.normalized_packages import NormalizedPackages -from src.v2.recommender import (RecommendationTask, GraphDB, License, - set_valid_cooccurrence_probability) -from src.v2.models import RecommenderRequest - -with open("tests/data/graph_response.json", "r") as f: - graph_resp = json.load(f) - -with open("tests/data/kronos_score_response.json", "r") as f: - insights_resp = json.load(f) - -with open("tests/data/kronos_score_comp_response.json", "r") as f: - insights_comp_resp = json.load(f) - -with open('tests/data/valid_license_analysis.json', 'r') as f: - license_resp = json.load(f) - -with open("tests/data/dependency_response.json", "r") as f: - dep_resp = json.load(f) - - -def mocked_requests_get(*args, **_kwargs): - """Mock the call to the insights service.""" - class MockResponse: - """Mock response object.""" - - def __init__(self, json_data, status_code): - """Create a mock json response.""" - self.json_data = json_data - self.status_code = status_code - - def json(self): - """Get the mock json response.""" - return self.json_data - - # return the URL to check whether we are calling the correct service. - print(args[0]) - return MockResponse({"url": args[0]}, 200) - - -class TestRecommendationTask(TestCase): - """Test for the recommendation task class.""" - - @mock.patch('requests.get', side_effect=mocked_requests_get) - @mock.patch('requests.Session.post', side_effect=mocked_requests_get) - def test_call_insights_recommender_npm(self, _mock_get, _mock_post): - """Test if the correct service is called for the correct ecosystem.""" - with mock.patch.dict('src.v2.recommender.os.environ', { - 'GOLANG_SERVICE_HOST': 'golang-insights', - 'CHESTER_SERVICE_HOST': 'npm-insights', - 'PYPI_SERVICE_HOST': 'pypi-insights', - 'PGM_SERVICE_HOST': 'pgm', - 'SERVICE_PORT': '6006', - 'HPF_SERVICE_HOST': 'hpf-insights', - }): - # Test whether the correct service is called for NPM. - called_url_json = RecommendationTask.call_insights_recommender([{"ecosystem": "npm"}]) - self.assertTrue('npm-insights' in called_url_json['url']) - # Test whether the correct service is called for PYPI. - called_url_json = RecommendationTask.call_insights_recommender([{ - "ecosystem": "pypi" - }]) - self.assertTrue('pypi-insights' in called_url_json['url']) - # Test whether the correct service is called for golang. - called_url_json = RecommendationTask.call_insights_recommender([{ - "ecosystem": "golang" - }]) - self.assertTrue('golang-insights' in called_url_json['url']) - # # Now test whether the correct service is called for maven. - called_url_json = RecommendationTask.call_insights_recommender( - [{"ecosystem": "maven", "package_list": ["org.slf4j:slf4j-api"]}]) - self.assertTrue('hpf-insights' in called_url_json['url']) - - -def mocked_response_graph(*args, **_kwargs): - """Mock the call to the insights service.""" - class MockResponse: - """Mock response object.""" - - def __init__(self, json_data, status_code): - """Create a mock json response.""" - self.json_data = json_data - self.status_code = status_code - - def json(self): - """Get the mock json response.""" - return self.json_data - - def raise_for_status(self): - if self.status_code != 200: - raise Exception('not 200') - - return MockResponse(graph_resp, 200) - - -def mocked_response_license(*args, **_kwargs): - """Mock the call to the execute.""" - class MockResponse: - """Mock response object.""" - - def __init__(self, json_data, status_code): - """Create a mock json response.""" - self.json_data = json_data - self.status_code = status_code - - def json(self): - """Get the mock json response.""" - return self.json_data - - if args and '6162' in str(args[0]): - return MockResponse(license_resp, 200) - else: - return MockResponse(dep_resp, 200) - - -@mock.patch('src.v2.recommender.persist_data_in_db') -@mock.patch('src.v2.recommender.RecommendationTask.call_insights_recommender', return_value=[]) -def test_execute(_mock_call_insights, _mock_db): - """Test the function execute.""" - with open("tests/v2/data/stack_aggregator_execute_input.json", "r") as f: - payload = json.load(f) - - r = RecommendationTask() - out = r.execute(arguments=payload, persist=False) - assert out['recommendation'] == "success" - r = RecommendationTask() - out = r.execute(arguments=payload, check_license=True, persist=False) - assert out['recommendation'] == "success" - - out = r.execute(arguments=payload, persist=True) - _mock_db.assert_called_once() - - -@mock.patch('src.v2.recommender.RecommendationTask.call_insights_recommender', - side_effect=[insights_comp_resp]) -@mock.patch('src.v2.recommender.GraphDB.get_version_information', - side_effect=[graph_resp['result']['data']]) -@mock.patch('src.v2.recommender.License.perform_license_analysis', - side_effect=license_resp) -def test_execute_with_insights(_mock1, _mock2, _mock3): - """Test the function execute.""" - with open("tests/v2/data/stack_aggregator_execute_input.json", "r") as f: - payload = json.load(f) - - r = RecommendationTask() - out = r.execute(arguments=payload, persist=False, check_license=False) - - assert out['recommendation'] == "success" - - -@mock.patch('src.v2.recommender.persist_data_in_db') -@mock.patch('src.v2.recommender.RecommendationTask.call_insights_recommender', return_value=[]) -def test_execute_empty_resolved(_mock_call_insights, _mock_db): - """Test the function execute.""" - with open("tests/v2/data/stack_aggregator_empty_resolved.json", "r") as f: - payload = json.load(f) - - r = RecommendationTask() - out = r.execute(arguments=payload, persist=False) - - assert out['recommendation'] == "success" - assert not out["result"]["companion"] - assert not out["result"]["usage_outliers"] - - r = RecommendationTask() - out = r.execute(arguments=payload, check_license=True, persist=False) - assert out['recommendation'] == "success" - - out = r.execute(arguments=payload, persist=True) - _mock_db.assert_called_once() - - -@mock.patch('src.v2.recommender.persist_data_in_db') -@mock.patch('src.v2.recommender.RecommendationTask.call_insights_recommender', return_value=[]) -def test_execute_both_resolved_type(_mock_call_insights, _mock_db): - """Test the function execute.""" - with open("tests/v2/data/stack_aggregator_combined_input.json", "r") as f: - payload = json.load(f) - - r = RecommendationTask() - out = r.execute(arguments=payload, persist=False) - assert out['recommendation'] == "success" - r = RecommendationTask() - out = r.execute(arguments=payload, check_license=True, persist=False) - assert out['recommendation'] == "success" - - out = r.execute(arguments=payload, persist=True) - _mock_db.assert_called_once() - - -def test_filter_versions(): - """Test the function filter_versions for latest version.""" - input_stack = {"io.vertx:vertx-web": "3.4.2", "io.vertx:vertx-core": "3.4.2"} - - with open("tests/data/companion_pkg_graph.json", "r") as f: - companion_packages_graph = json.load(f) - - g = GraphDB() - filtered_comp_packages_graph, filtered_list = g.filter_versions(companion_packages_graph, - input_stack) - assert len(filtered_comp_packages_graph) > 0 - assert len(filtered_list) > 0 - - -def test_prepare_final_filtered_list(): - """Test the function filter_versions.""" - with open("tests/data/companion_pkg_graph_deps.json", "r") as f: - comp_pkg_graph = json.load(f) - deps_pkg_graph = comp_pkg_graph['deps'] - release_pkg_graph = comp_pkg_graph['gh_release'] - - g = GraphDB() - filtered_list = g.prepare_final_filtered_list(deps_pkg_graph) - assert len(filtered_list) > 0 - - filtered_list = g.prepare_final_filtered_list(release_pkg_graph) - assert len(filtered_list) > 0 - - -@mock.patch('requests.Session.post', side_effect=mocked_response_graph) -def test_get_version_information(_mock1): - """Test the function get_version_information.""" - out = GraphDB().get_version_information(['io.vertx:vertx-web'], 'maven') - assert len(out) == 1 - - -def test_get_topics(): - """Test the function get topics.""" - comp_list = GraphDB.get_topics_for_comp(graph_resp['result']['data'], - insights_resp[0]['companion_packages']) - - assert comp_list is not None - assert isinstance(comp_list, list) - - -@mock.patch('src.v2.recommender.extract_user_stack_package_licenses', return_value=[]) -@mock.patch('requests.Session.post', side_effect=mocked_response_license) -def test_perform_license_analysis(_mock1, _mock2): - """Test license analysis function.""" - with open("tests/v2/data/license_analysis.json", "r") as f: - payload = json.load(f) - request = RecommenderRequest(**payload) - comp_graph = License.perform_license_analysis( - packages=NormalizedPackages(request.packages, 'maven'), - filtered_comp_packages_graph=payload['filtered_comp_packages_graph'], - filtered_companion_packages=payload['filtered_companion_packages'], - external_request_id=payload['external_request_id']) - - assert comp_graph is not None - - -@mock.patch('src.v2.recommender.License.invoke_license_analysis_service', - return_value={'status': 'successful', 'license_filter': {}}) -def test_apply_license_filter(_mock1): - """Test the function apply_license_filter.""" - with open('tests/data/epv_list.json', 'r') as f: - resp = json.load(f) - - out = License.apply_license_filter(None, resp) - assert isinstance(out, dict) - - -def test_set_valid_cooccurrence_probability(): - """Test the function set_valid_cooccurrence_probability.""" - input = [{"ecosystem": "maven", "name": "io.fabric8.funktion.connector:connector-smpp", - "cooccurrence_probability": 'nan'}] - components = set_valid_cooccurrence_probability(input) - for component in components: - assert component['cooccurrence_probability'] == 100 - - -if __name__ == '__main__': - test_execute() - test_execute_with_insights() - test_filter_versions() - test_get_version_information() - test_apply_license_filter() - test_perform_license_analysis() - test_get_topics() - test_prepare_final_filtered_list() diff --git a/tests/v2/test_recommender_api.py b/tests/v2/test_recommender_api.py new file mode 100644 index 00000000..894577c7 --- /dev/null +++ b/tests/v2/test_recommender_api.py @@ -0,0 +1,327 @@ +"""Tests for the REST API of the backbone service.""" +import json +from unittest import mock + +from src.v2.models import RecommenderRequest, Package +from tests.v2.data import ( + npm_insights_response_for_request, + graph_package_response_for_npm_express_winston_moment, +) + + +def get_json_from_response(response): + """Decode JSON from response.""" + response = json.loads(response.data.decode("utf8")) + assert response["result"]["_audit"] is not None + # delete _audit field as it has time value which is not assertable + del response["result"]["_audit"] + return response + + +def test_invalid_request_payload(client): + """Test invalid request payload.""" + response = client.post( + "/api/v2/recommender", data=json.dumps({}), content_type="application/json" + ) + assert response.status_code == 500 + + +def test_golang_empty_response(client): + """Test golang ecosystem, it should return empty result.""" + response = client.post( + "/api/v2/recommender", + data=json.dumps( + RecommenderRequest( + registration_status="", + external_request_id="", + ecosystem="golang", + manifest_file_path="", + packages=[], + ).dict() + ), + content_type="application/json", + ) + assert response.status_code == 200 + json_val = json.loads(response.data.decode("utf8")) + assert json_val == {} + + +class _Response: + """Mock Response.""" + + def __init__(self, code, json): + self.status_code = code + self._json = json + + def json(self): + """Json Response.""" + return self._json + + def raise_for_status(self): + if self.status_code != 200: + raise Exception("not 200") + + +@mock.patch("src.v2.recommender.post_gremlin", return_value={"result": {"data": []}}) +@mock.patch("requests.Session.post", return_value=_Response(404, {})) +def test_recommendation_response_with_insight_error( + _mock_insight_response, _mock_gremlin, client +): + """Test golang ecosystem, it should return empty result.""" + response = client.post( + "/api/v2/recommender?persist=false", + data=json.dumps( + RecommenderRequest( + registration_status="", + external_request_id="", + ecosystem="npm", + manifest_file_path="", + packages=[Package(name="request", version="2.88.2")], + ).dict() + ), + content_type="application/json", + ) + assert response.status_code == 500 + + +@mock.patch("src.v2.recommender.post_gremlin", return_value={"result": {"data": []}}) +@mock.patch("requests.Session.post", return_value=_Response(404, {})) +def test_recommendation_response_with_empty_package_request( + _mock_insight_response, _mock_gremlin, client +): + """Test with empty package list.""" + response = client.post( + "/api/v2/recommender?persist=false", + data=json.dumps( + RecommenderRequest( + registration_status="FREETIER", + external_request_id="foo", + ecosystem="npm", + manifest_file_path="/foo.bar", + packages=[], + ).dict() + ), + content_type="application/json", + ) + assert response.status_code == 500 + + +@mock.patch("src.v2.recommender.post_gremlin", return_value={"result": {"data": []}}) +@mock.patch( + "requests.Session.post", + return_value=_Response( + 200, + [ + { + "companion_packages": [], + "ecosystem": "npm", + "missing_packages": ["blala"], + "package_to_topic_dict": {}, + } + ], + ), +) +@mock.patch("src.v2.recommender.persist_data_in_db") +def test_recommendation_response_with_empty_insight_response( + _mock_insight_response, _mock_gremlin, _mock_persist_db, client +): + """Test with empty recommender response.""" + response = client.post( + "/api/v2/recommender?persist=true", + data=json.dumps( + RecommenderRequest( + registration_status="FREETIER", + external_request_id="foo", + ecosystem="npm", + manifest_file_path="/foo.bar", + packages=[Package(name="blala", version="1.000")], + ).dict() + ), + content_type="application/json", + ) + assert response.status_code == 200 + response = get_json_from_response(response) + assert response == { + "external_request_id": "foo", + "recommendation": "success", + "result": { + "companion": [], + "external_request_id": "foo", + "manifest_file_path": "/foo.bar", + "manifest_name": None, + "recommendation_status": "success", + "registration_status": "FREETIER", + "usage_outliers": [], + "uuid": None, + }, + } + + +@mock.patch("src.v2.recommender.post_gremlin", return_value={"result": {"data": []}}) +@mock.patch( + "requests.Session.post", + return_value=_Response( + 200, + npm_insights_response_for_request.DATA, + ), +) +def test_recommendation_response_with_empty_graph( + _mock_insight_response, _mock_gremlin, client +): + """Test with empty graph response.""" + response = client.post( + "/api/v2/recommender?persist=false", + data=json.dumps( + RecommenderRequest( + registration_status="FREETIER", + external_request_id="foo", + ecosystem="npm", + manifest_file_path="/foo.bar", + packages=[Package(name="request", version="2.88.2")], + ).dict() + ), + content_type="application/json", + ) + assert response.status_code == 200 + response = get_json_from_response(response) + assert response["result"] == { + "companion": [], + "external_request_id": "foo", + "manifest_file_path": "/foo.bar", + "manifest_name": None, + "recommendation_status": "success", + "registration_status": "FREETIER", + "usage_outliers": [], + "uuid": None, + } + + +@mock.patch( + "src.v2.recommender.post_gremlin", + return_value=graph_package_response_for_npm_express_winston_moment.DATA, +) +@mock.patch( + "requests.Session.post", + return_value=_Response( + 200, + npm_insights_response_for_request.DATA, + ), +) +def test_recommendation_response_with_2_packages_from_graph( + _mock_insight_response, _mock_gremlin, client +): + """Test for happy scenario.""" + response = client.post( + "/api/v2/recommender?persist=false", + data=json.dumps( + RecommenderRequest( + registration_status="FREETIER", + external_request_id="foo", + ecosystem="npm", + manifest_file_path="/foo.bar", + packages=[Package(name="request", version="2.88.2")], + ).dict() + ), + content_type="application/json", + ) + assert response.status_code == 200 + response = get_json_from_response(response) + assert response["result"] == { + "companion": [ + { + "cooccurrence_count": 0, + "cooccurrence_probability": 52.50463667254613, + "dependencies": None, + "ecosystem": "npm", + "github": { + "contributors": "238", + "dependent_projects": "48988", + "dependent_repos": "893994", + "forks_count": "8571", + "issues": { + "month": {"closed": 18, "opened": 18}, + "year": {"closed": -1, "opened": -1}, + }, + "latest_release_duration": "2018-10-27 03:12:11", + "open_issues_count": "155", + "pull_requests": { + "month": {"closed": 0, "opened": 3}, + "year": {"closed": 61, "opened": 75}, + }, + "size": "N/A", + "stargazers_count": "51280", + "total_releases": "291", + "used_by": [ + {"name": "Automattic/mongoose", "stars": "14372"}, + {"name": "ReactTraining/react-router", "stars": "27232"}, + {"name": "angular/angular.js", "stars": "57749"}, + {"name": "angular/angular", "stars": "31547"}, + {"name": "facebook/jest", "stars": "14465"}, + {"name": "facebook/react", "stars": "84669"}, + {"name": "postcss/autoprefixer", "stars": "14331"}, + {"name": "reactjs/redux", "stars": "36922"}, + {"name": "socketio/socket.io", "stars": "38209"}, + {"name": "webpack/webpack", "stars": "35429"}, + ], + "watchers": "1799", + }, + "latest_version": "4.17.1", + "licenses": [], + "name": "express", + "topic_list": ["accepts"], + "url": "https://snyk.io/vuln/npm:express", + "version": "4.17.1", + }, + { + "cooccurrence_count": 0, + "cooccurrence_probability": 51.04134789351682, + "dependencies": None, + "ecosystem": "npm", + "github": { + "contributors": "30", + "dependent_projects": "6413", + "dependent_repos": "35152", + "forks_count": "1129", + "issues": { + "month": {"closed": -1, "opened": -1}, + "year": {"closed": -1, "opened": -1}, + }, + "latest_release_duration": "2018-06-12 16:31:26", + "open_issues_count": "236", + "pull_requests": { + "month": {"closed": -1, "opened": -1}, + "year": {"closed": -1, "opened": -1}, + }, + "size": "N/A", + "stargazers_count": "12020", + "total_releases": "73", + "used_by": [ + {"name": "amir20/phantomjs-node", "stars": "2884"}, + {"name": "angular/angular.js", "stars": "56712"}, + {"name": "angular/angular", "stars": "26987"}, + {"name": "angular/material2", "stars": "10337"}, + {"name": "apidoc/apidoc", "stars": "4685"}, + {"name": "cyclejs/cyclejs", "stars": "7180"}, + {"name": "flatiron/prompt", "stars": "1142"}, + {"name": "foreverjs/forever", "stars": "9868"}, + {"name": "ionic-team/ionic-native", "stars": "1384"}, + {"name": "ionic-team/ionic", "stars": "30889"}, + ], + "watchers": "224", + }, + "latest_version": "3.2.1", + "licenses": [], + "name": "winston", + "topic_list": ["async"], + "url": "https://snyk.io/vuln/npm:winston", + "version": "3.2.1", + }, + ], + "external_request_id": "foo", + "manifest_file_path": "/foo.bar", + "manifest_name": None, + "recommendation_status": "success", + "registration_status": "FREETIER", + "usage_outliers": [], + "uuid": None, + }