diff --git a/openshift/template.yaml b/openshift/template.yaml index dfef102c..636e47ed 100644 --- a/openshift/template.yaml +++ b/openshift/template.yaml @@ -26,10 +26,8 @@ objects: - command: - /bin/entrypoint.sh env: - - name: BAYESIAN_GREMLIN_HTTP_SERVICE_HOST - value: bayesian-gremlin-http - - name: BAYESIAN_GREMLIN_HTTP_SERVICE_PORT - value: "8182" + - name: GREMLIN_URL + value: "http://bayesian-gremlin-http:8182" - name: DEPLOYMENT_PREFIX valueFrom: configMapKeyRef: @@ -46,22 +44,14 @@ objects: value: bayesian-pgbouncer - name: PGBOUNCER_SERVICE_PORT value: "5432" - - name: LICENSE_SERVICE_HOST - value: f8a-license-analysis - - name: LICENSE_SERVICE_PORT - value: "6162" - - name: CHESTER_SERVICE_HOST - value: f8a-npm-insights - - name: PYPI_SERVICE_HOST - value: f8a-pypi-insights - - name: GOLANG_SERVICE_HOST - value: f8a-golang-insights - - name: MAVEN_SERVICE_HOST - value: f8a-hpf-insights-maven - - name: HPF_SERVICE_HOST - value: f8a-hpf-insights - - name: SERVICE_PORT - value: "6006" + - name: LICENSE_ANALYSIS_BASE_URL + value: "http://f8a-license-analysis:6162" + - name: NPM_INSIGHTS_BASE_URL + value: "http://f8a-npm-insights:6006" + - name: PYPI_INSIGHTS_BASE_URL + value: "http://f8a-pypi-insights:6006" + - name: MAVEN_INSIGHTS_BASE_URL + value: "http://f8a-hpf-insights-maven:6006" - name: POSTGRESQL_DATABASE valueFrom: secretKeyRef: diff --git a/src/recommender.py b/src/recommender.py index ae999b93..5205d1cd 100644 --- a/src/recommender.py +++ b/src/recommender.py @@ -70,8 +70,10 @@ import re import logging +from pydantic import AnyHttpUrl + +from src.settings import AGGREGATOR_SETTINGS, RECOMMENDER_SETTINGS, SETTINGS from src.utils import (create_package_dict, get_session_retry, select_latest_version, - GREMLIN_SERVER_URL_REST, LICENSE_SCORING_URL_REST, convert_version_to_proper_semantic, get_response_data, version_info_tuple, persist_data_in_db, post_http_request) from src.stack_aggregator import extract_user_stack_package_licenses @@ -112,7 +114,7 @@ def get_version_information(input_list, ecosystem): } # Query Gremlin with packages list to get their version information - gremlin_response = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + gremlin_response = post_http_request(url=SETTINGS.gremlin_url, payload=payload) if gremlin_response is None: return [] response = get_response_data(gremlin_response, [{0: 0}]) @@ -289,7 +291,7 @@ class License: @staticmethod def invoke_license_analysis_service(user_stack_packages, alt_packages, comp_packages): """Pass given args to stack_license analysis.""" - license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license" + license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license" payload = { "packages": user_stack_packages, @@ -413,41 +415,20 @@ def set_valid_cooccurrence_probability(package_list=[]): return new_package_list -class RecommendationTask: - """Recommendation task.""" +def _prepare_insights_url(base_url: AnyHttpUrl) -> AnyHttpUrl: + assert base_url + return "{url}/api/v1/companion_recommendation".format(url=base_url) - _analysis_name = 'recommendation_v2' - description = 'Get Recommendation' - kronos_ecosystems = ['maven'] - chester_ecosystems = ['npm'] - hpf_ecosystems = ['maven'] - pypi_ecosystems = ['pypi'] - golang_ecosystem = ['golang'] - @staticmethod - def get_insights_url(payload): - """Get the insights url based on the ecosystem.""" - if payload and 'ecosystem' in payload[0]: - if payload[0]['ecosystem'] in RecommendationTask.chester_ecosystems: - INSIGHTS_SERVICE_HOST = os.getenv("CHESTER_SERVICE_HOST") - elif payload[0]['ecosystem'] in RecommendationTask.pypi_ecosystems: - INSIGHTS_SERVICE_HOST = os.getenv("PYPI_SERVICE_HOST") - elif payload[0]['ecosystem'] in RecommendationTask.golang_ecosystem: - INSIGHTS_SERVICE_HOST = os.environ.get("GOLANG_SERVICE_HOST") - else: - INSIGHTS_SERVICE_HOST = os.getenv("HPF_SERVICE_HOST") + "-" + payload[0][ - 'ecosystem'] +ECOSYSTEM_TO_INSIGHTS_URL = { + "pypi": _prepare_insights_url(RECOMMENDER_SETTINGS.pypi_insights_base_url), + "npm": _prepare_insights_url(RECOMMENDER_SETTINGS.npm_insights_base_url), + "maven": _prepare_insights_url(RECOMMENDER_SETTINGS.maven_insights_base_url), +} - INSIGHTS_URL_REST = "http://{host}:{port}".format(host=INSIGHTS_SERVICE_HOST, - port=os.getenv("SERVICE_PORT")) - insights_url = INSIGHTS_URL_REST + "/api/v1/companion_recommendation" - - return insights_url - - else: - logger.error('Payload information not passed in the call, Quitting! inights ' - 'recommender\'s call') +class RecommendationTask: + """Recommendation task.""" @staticmethod def call_insights_recommender(payload): @@ -459,7 +440,8 @@ def call_insights_recommender(payload): try: # TODO remove hardcodedness for payloads with multiple ecosystems - insights_url = RecommendationTask.get_insights_url(payload) + insights_url = ECOSYSTEM_TO_INSIGHTS_URL.get(payload[0]['ecosystem'], None) + assert insights_url response = get_session_retry().post(insights_url, json=payload) if response.status_code != 200: @@ -519,13 +501,6 @@ def execute(self, arguments=None, persist=True, check_license=False): 'comp_package_count_threshold': int(os.environ.get( 'MAX_COMPANION_PACKAGES', 5)) } - if details['ecosystem'] in self.kronos_ecosystems: - insights_payload.update({ - 'alt_package_count_threshold': int(os.environ.get('MAX_ALTERNATE_PACKAGES', 2)), - 'outlier_probability_threshold': float(os.environ.get('OUTLIER_THRESHOLD', - 0.6)), - 'user_persona': "1", # TODO - remove janus hardcoded value - }) input_task_for_insights_recommender = [insights_payload] # Call PGM and get the response diff --git a/src/rest_api.py b/src/rest_api.py index afb433bd..856e13d7 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -54,7 +54,7 @@ def _recommender(handler): input_json = request.get_json() external_request_id = input_json['external_request_id'] - logger.info('%s recommender/ request with payload: %s', external_request_id, input_json) + logger.info('%s recommender/ request', external_request_id) try: check_license = request.args.get('check_license', 'false') == 'true' @@ -96,8 +96,7 @@ def _stack_aggregator(handler): } external_request_id = input_json['external_request_id'] - logger.info('%s stack_aggregator/ request with payload: %s', - external_request_id, input_json) + logger.info('%s stack_aggregator/ request', external_request_id) try: persist = request.args.get('persist', 'true') == 'true' diff --git a/src/settings.py b/src/settings.py index 9f6e60c7..f3158f6c 100644 --- a/src/settings.py +++ b/src/settings.py @@ -2,24 +2,34 @@ from typing import Dict -from pydantic import BaseSettings, HttpUrl +from pydantic import BaseSettings, HttpUrl, AnyHttpUrl class AggregatorSettings(BaseSettings): """Create Settings from env.""" - snyk_package_url_format: HttpUrl = "https://snyk.io/vuln/{ecosystem}:{package}" + license_analysis_base_url: AnyHttpUrl = "http://f8a-license-analysis:6162" snyk_signin_url: HttpUrl = "https://snyk.io/login" + snyk_package_url_format: HttpUrl = "https://snyk.io/vuln/{ecosystem}:{package}" snyk_ecosystem_map: Dict[str, str] = {"pypi": "pip"} - disable_unknown_package_flow: bool = False class RecommenderSettings(BaseSettings): """Create Recommender Settings from env.""" - chester_service_host: str = "notset" - pypi_service_host: str = "notset" - maven_service_host: str = "notset" - service_port: int = 6006 + npm_insights_base_url: AnyHttpUrl = "http://f8a-npm-insights:6006" + pypi_insights_base_url: AnyHttpUrl = "http://f8a-pypi-insights:6006" + maven_insights_base_url: AnyHttpUrl = "http://f8a-hpf-insights-maven:6006" unknown_packages_threshold: float = 0.3 max_companion_packages: int = 5 + + +class Settings(BaseSettings): + """General settings.""" + + gremlin_url: AnyHttpUrl = "http://bayesian-gremlin-http:8182" + + +SETTINGS = Settings() +RECOMMENDER_SETTINGS = RecommenderSettings() +AGGREGATOR_SETTINGS = AggregatorSettings() diff --git a/src/stack_aggregator.py b/src/stack_aggregator.py index f5aaf54c..7a7c5db3 100644 --- a/src/stack_aggregator.py +++ b/src/stack_aggregator.py @@ -12,9 +12,9 @@ import requests import copy from collections import defaultdict -from src.settings import AggregatorSettings -from src.utils import (select_latest_version, server_create_analysis, LICENSE_SCORING_URL_REST, - post_http_request, GREMLIN_SERVER_URL_REST, persist_data_in_db, +from src.settings import AGGREGATOR_SETTINGS, SETTINGS +from src.utils import (select_latest_version, server_create_analysis, + post_http_request, persist_data_in_db, GREMLIN_QUERY_SIZE, format_date) import logging @@ -27,7 +27,7 @@ def get_recommended_version(ecosystem, name, version): ".out('has_version').not(out('has_cve')).values('version');"\ .format(eco=ecosystem, pkg=name) payload = {'gremlin': query} - result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + result = post_http_request(url=SETTINGS.gremlin_url, payload=payload) if result: versions = result['result']['data'] if len(versions) == 0: @@ -269,7 +269,7 @@ def _extract_license_outliers(license_service_output): def perform_license_analysis(license_score_list, dependencies): """Pass given license_score_list to stack_license analysis and process response.""" - license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license" + license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license" payload = { "packages": license_score_list @@ -510,7 +510,7 @@ def get_tr_dependency_data(epv_set): i = 1 # call_gremlin in batch payload = {'gremlin': query} - result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + result = post_http_request(url=SETTINGS.gremlin_url, payload=payload) if result: tr_epv_list['result']['data'] += result['result']['data'] query = "epv=[];" @@ -519,7 +519,7 @@ def get_tr_dependency_data(epv_set): if i > 1: payload = {'gremlin': query} time_start = time.time() - result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + result = post_http_request(url=SETTINGS.gremlin_url, payload=payload) logger.info('elapsed_time for gremlin call: {}'.format(time.time() - time_start)) if result: tr_epv_list['result']['data'] += result['result']['data'] @@ -571,7 +571,7 @@ def get_dependency_data(epv_set): i = 1 # call_gremlin in batch payload = {'gremlin': query} - result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + result = post_http_request(url=SETTINGS.gremlin_url, payload=payload) if result: epv_list['result']['data'] += result['result']['data'] query = "epv=[];" @@ -579,7 +579,7 @@ def get_dependency_data(epv_set): if i > 1: payload = {'gremlin': query} - result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + result = post_http_request(url=SETTINGS.gremlin_url, payload=payload) if result: epv_list['result']['data'] += result['result']['data'] @@ -663,16 +663,13 @@ def execute(aggregated=None, persist=True): 'result': stack_data} # Ingestion of Unknown dependencies logger.info("Unknown ingestion flow process initiated.") - if AggregatorSettings().disable_unknown_package_flow: - logger.warning('Skipping unknown flow %s', unknown_dep_list) - else: - try: - for dep in unknown_dep_list: - server_create_analysis(ecosystem, dep['name'], dep['version'], api_flow=True, - force=False, force_graph_sync=True) - except Exception as e: - logger.error('Ingestion has been failed for ' + dep['name']) - logger.error(e) - pass + try: + for dep in unknown_dep_list: + server_create_analysis(ecosystem, dep['name'], dep['version'], api_flow=True, + force=False, force_graph_sync=True) + except Exception as e: + logger.error('Ingestion has been failed for ' + dep['name']) + logger.error(e) + pass return persiststatus diff --git a/src/utils.py b/src/utils.py index 608fb79b..7f6076ad 100644 --- a/src/utils.py +++ b/src/utils.py @@ -22,6 +22,8 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import sessionmaker +from src.settings import SETTINGS + logger = logging.getLogger(__name__) @@ -39,13 +41,6 @@ class RequestException(Exception): logger = logging.getLogger(__file__) -GREMLIN_SERVER_URL_REST = "http://{host}:{port}".format( - host=os.environ.get("BAYESIAN_GREMLIN_HTTP_SERVICE_HOST", "localhost"), - port=os.environ.get("BAYESIAN_GREMLIN_HTTP_SERVICE_PORT", "8182")) - -LICENSE_SCORING_URL_REST = "http://{host}:{port}".format( - host=os.environ.get("LICENSE_SERVICE_HOST", "localhost"), - port=os.environ.get("LICENSE_SERVICE_PORT", "6162")) zero_version = sv.Version("0.0.0") # Create Postgres Connection Session @@ -104,7 +99,7 @@ def get_osio_user_count(ecosystem, name, version): 'gremlin': str_gremlin } - json_response = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) + json_response = post_http_request(url=SETTINGS.gremlin_url, payload=payload) return json_response.get('result').get('data', ['-1'])[0] @@ -330,7 +325,7 @@ def post_gremlin(query: str, bindings: Dict = None) -> Dict: if bindings: payload['bindings'] = bindings try: - response = get_session_retry().post(url=GREMLIN_SERVER_URL_REST, json=payload) + response = get_session_retry().post(url=SETTINGS.gremlin_url, json=payload) response.raise_for_status() except Exception as e: logger.error(traceback.format_exc()) diff --git a/src/v2/license_service.py b/src/v2/license_service.py index 1b6c6c16..8e411efd 100644 --- a/src/v2/license_service.py +++ b/src/v2/license_service.py @@ -3,7 +3,8 @@ import logging from typing import List, Set -from src.utils import LICENSE_SCORING_URL_REST, post_http_request +from src.utils import post_http_request +from src.settings import AGGREGATOR_SETTINGS from src.v2.models import LicenseAnalysis, PackageDetails @@ -158,7 +159,7 @@ def get_license_service_request_payload(package_details: List[PackageDetails]): def get_license_analysis_for_stack( package_details: List[PackageDetails]) -> LicenseAnalysis: # pylint:disable=R0914 """Create LicenseAnalysis from license server.""" - license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license" + license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license" # form payload for license service request payload = { diff --git a/src/v2/recommender.py b/src/v2/recommender.py index bdcdc6c4..2db63a82 100644 --- a/src/v2/recommender.py +++ b/src/v2/recommender.py @@ -9,12 +9,12 @@ import time from typing import List, Dict -from pydantic import BaseModel +from pydantic import BaseModel, AnyHttpUrl from src.utils import get_session_retry, persist_data_in_db, post_gremlin from src.v2.stack_aggregator import get_github_details, get_snyk_package_link from src.v2.normalized_packages import NormalizedPackages -from src.settings import RecommenderSettings +from src.settings import RECOMMENDER_SETTINGS from src.v2.models import ( RecommenderRequest, @@ -33,9 +33,9 @@ class InsightsRequest(BaseModel): transitive_stack: List[str] = [] package_list: List[str] unknown_packages_ratio_threshold: float = ( - RecommenderSettings().unknown_packages_threshold + RECOMMENDER_SETTINGS.unknown_packages_threshold ) - comp_package_count_threshold: int = RecommenderSettings().max_companion_packages + comp_package_count_threshold: int = RECOMMENDER_SETTINGS.max_companion_packages class InsightsCallException(Exception): @@ -46,19 +46,15 @@ class InsightsWithEmptyPackageException(Exception): """Exception for empty request packages.""" -def _prepare_insights_url(host: str) -> str: - assert host - url = "http://{host}:{port}".format( - host=host, port=RecommenderSettings().service_port - ) - endpoint = "{url}/api/v1/companion_recommendation".format(url=url) - return endpoint +def _prepare_insights_url(base_url: AnyHttpUrl) -> AnyHttpUrl: + assert base_url + return "{url}/api/v1/companion_recommendation".format(url=base_url) ECOSYSTEM_TO_INSIGHTS_URL = { - "pypi": _prepare_insights_url(RecommenderSettings().pypi_service_host), - "npm": _prepare_insights_url(RecommenderSettings().chester_service_host), - "maven": _prepare_insights_url(RecommenderSettings().maven_service_host), + "pypi": _prepare_insights_url(RECOMMENDER_SETTINGS.pypi_insights_base_url), + "npm": _prepare_insights_url(RECOMMENDER_SETTINGS.npm_insights_base_url), + "maven": _prepare_insights_url(RECOMMENDER_SETTINGS.maven_insights_base_url), } diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py index b976e767..2b043c61 100644 --- a/src/v2/stack_aggregator.py +++ b/src/v2/stack_aggregator.py @@ -15,7 +15,7 @@ from f8a_utils.gh_utils import GithubUtils from f8a_utils.ingestion_utils import unknown_package_flow -from src.settings import AggregatorSettings +from src.settings import AGGREGATOR_SETTINGS from src.utils import (select_latest_version, persist_data_in_db, post_gremlin, GREMLIN_QUERY_SIZE, format_date) @@ -173,9 +173,9 @@ def _has_vulnerability(pkg: PackageDetails) -> bool: # (fixme) link to snyk package should be identified during ingestion. def get_snyk_package_link(ecosystem: str, package: str) -> str: """Prepare snyk package link based on ecosystem and package name.""" - ecosystem = AggregatorSettings().snyk_ecosystem_map.get(ecosystem, ecosystem) - return AggregatorSettings().snyk_package_url_format.format(ecosystem=ecosystem, - package=quote(package, safe='')) + ecosystem = AGGREGATOR_SETTINGS.snyk_ecosystem_map.get(ecosystem, ecosystem) + return AGGREGATOR_SETTINGS.snyk_package_url_format.format(ecosystem=ecosystem, + package=quote(package, safe='')) class Aggregator: @@ -304,7 +304,7 @@ def get_result(self) -> StackAggregatorResult: analyzed_dependencies=package_details, unknown_dependencies=unknown_dependencies, license_analysis=license_analysis, - registration_link=AggregatorSettings().snyk_signin_url) + registration_link=AGGREGATOR_SETTINGS.snyk_signin_url) def initiate_unknown_package_ingestion(self): """Ingestion of Unknown dependencies.""" @@ -381,20 +381,6 @@ def __init__(self, request: StackAggregatorRequest = None, self._normalized_packages = normalized_packages self.filtered_vul = {} - def initiate_unknown_package_ingestion(self): - """Ingestion of Unknown dependencies.""" - ecosystem = self._normalized_packages.ecosystem - pkg_list = self.get_all_unknown_packages() - unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name, - version=pkg.version), pkg_list)) - try: - unknown_package_flow(ecosystem, unknown_pkgs) - except Exception as e: - logger.error('Unknown ingestion failed with %s', e) - else: - logger.debug('Unknown ingestion executed for %s packages in %s ecosystem', - len(pkg_list), ecosystem) - def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]: """Get package data from graph along with vulnerability.""" get_package_details_with_vul_query = """ diff --git a/tests/test_recommender.py b/tests/test_recommender.py index 54554fef..f447d89c 100644 --- a/tests/test_recommender.py +++ b/tests/test_recommender.py @@ -1,5 +1,4 @@ """Tests for the recommender module.""" -from unittest import TestCase from unittest import mock import json import logging @@ -43,40 +42,6 @@ def json(self): return MockResponse({"url": args[0]}, 200) -class TestRecommendationTask(TestCase): - """Test for the recommendation task class.""" - - @mock.patch('requests.get', side_effect=mocked_requests_get) - @mock.patch('requests.Session.post', side_effect=mocked_requests_get) - def test_call_insights_recommender_npm(self, _mock_get, _mock_post): - """Test if the correct service is called for the correct ecosystem.""" - with mock.patch.dict('src.recommender.os.environ', { - 'GOLANG_SERVICE_HOST': 'golang-insights', - 'CHESTER_SERVICE_HOST': 'npm-insights', - 'PYPI_SERVICE_HOST': 'pypi-insights', - 'PGM_SERVICE_HOST': 'pgm', - 'SERVICE_PORT': '6006', - 'HPF_SERVICE_HOST': 'hpf-insights', - }): - # Test whether the correct service is called for NPM. - called_url_json = RecommendationTask.call_insights_recommender([{"ecosystem": "npm"}]) - self.assertTrue('npm-insights' in called_url_json['url']) - # Test whether the correct service is called for PYPI. - called_url_json = RecommendationTask.call_insights_recommender([{ - "ecosystem": "pypi" - }]) - self.assertTrue('pypi-insights' in called_url_json['url']) - # Test whether the correct service is called for golang. - called_url_json = RecommendationTask.call_insights_recommender([{ - "ecosystem": "golang" - }]) - self.assertTrue('golang-insights' in called_url_json['url']) - # Now test whether the correct service is called for maven. - called_url_json = RecommendationTask.call_insights_recommender( - [{"ecosystem": "maven", "package_list": ["org.slf4j:slf4j-api"]}]) - self.assertTrue('hpf-insights' in called_url_json['url']) - - def mocked_response_graph(*args, **_kwargs): """Mock the call to the insights service.""" class MockResponse: diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index dab43566..fbfd05d0 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -114,16 +114,6 @@ def test_recommendation_api_endpoint(_mock_object, client): assert jsn['external_request_id'] is not None -@mock.patch('requests.Session.post', return_value=Response2) -def test_recommendation_api_endpoint_exception(_mock_object, client): - """Check the /recommender REST API endpoint.""" - rec_resp = client.post(api_route_for("recommender"), - data=json.dumps(payload), content_type='application/json') - jsn = get_json_from_response(rec_resp) - assert jsn['recommendation'] == 'pgm_error' - assert jsn['external_request_id'] == payload['external_request_id'] - - if __name__ == '__main__': test_readiness_endpoint() test_liveness_endpoint() diff --git a/tests/test_stack_aggregator.py b/tests/test_stack_aggregator.py index 9c46d090..c7c75055 100644 --- a/tests/test_stack_aggregator.py +++ b/tests/test_stack_aggregator.py @@ -76,26 +76,6 @@ def test_execute(_mock_get, _mock_post): assert out['result']['stack_data'][0]['user_stack_info']['transitive_count'] == -1 -@mock.patch('requests.get', side_effect=mock_dependency_response) -@mock.patch('requests.Session.post', side_effect=mock_dependency_response) -@mock.patch('src.stack_aggregator.server_create_analysis') -def test_execute_unknown_with_disable_unknown_package_flow(_mock_unknown, _mock_get, - _mock_post, monkeypatch): - """Test the function execute.""" - with open("tests/data/stack_aggregator_execute_input.json", "r") as f: - payload = json.loads(f.read()) - - # add unknown package as direct dependency - payload['result'][0]['details'][0]['_resolved'].append({'package': 'six', 'version': '3.2.1'}) - - monkeypatch.setenv('DISABLE_UNKNOWN_PACKAGE_FLOW', '1') - s = stack_aggregator.StackAggregator() - out = s.execute(payload, False) - assert out['stack_aggregator'] == "success" - assert out['result']['stack_data'][0]['user_stack_info']['transitive_count'] == -1 - _mock_unknown.assert_not_called() - - def mock_licenses_resp_component_conflict(*_args, **_kwargs): """Mock the call to the insights service.""" class MockResponse: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5a7fd2c4..acd5682b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,12 +5,13 @@ from unittest import mock from pytest import raises from tests.test_rest_api import response +from src.settings import SETTINGS from src.utils import push_data, get_time_delta from src.utils import ( - convert_version_to_proper_semantic as cvs, GREMLIN_SERVER_URL_REST, format_date, + convert_version_to_proper_semantic as cvs, format_date, version_info_tuple as vt, select_latest_version as slv, get_osio_user_count, create_package_dict, post_http_request, - server_create_analysis, select_from_db, total_time_elapsed, post_gremlin, + select_from_db, total_time_elapsed, post_gremlin, GremlinExeception, RequestException) METRICS_COLLECTION_URL = "http://{base_url}:{port}/api/v1/prometheus".format( @@ -168,13 +169,7 @@ def test_post_http_request(_mock1): """Test error response for gremlin.""" payload = {'gremlin': ''} with raises(RequestException): - post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload) - - -def test_server_create_analysis(): - """Test server_create_analysis.""" - rec_resp = server_create_analysis("npm", "hjhjhjh", "1.1") - assert rec_resp is None + post_http_request(url=SETTINGS.gremlin_url, payload=payload) def test_select_from_db(): @@ -252,4 +247,3 @@ def test_post_gremlin_normal(_mock_post): test_get_osio_user_count() test_post_http_request() test_create_package_dict() - test_server_create_analysis() diff --git a/tests/v2/test_stack_aggregator.py b/tests/v2/test_stack_aggregator.py index 3b61f21c..623b7ff6 100644 --- a/tests/v2/test_stack_aggregator.py +++ b/tests/v2/test_stack_aggregator.py @@ -74,13 +74,11 @@ def _go_request_body(): @mock.patch('src.v2.stack_aggregator.post_gremlin') @mock.patch('src.v2.stack_aggregator.get_license_analysis_for_stack') -def test_with_2_public_vuln(_mock_license, _mock_gremlin, monkeypatch): +def test_with_2_public_vuln(_mock_license, _mock_gremlin): """Test basic request and response.""" with open("tests/v2/data/graph_response_2_public_vuln.json", "r") as fin: _mock_gremlin.return_value = json.load(fin) - monkeypatch.setenv('SNYK_PACKAGE_URL_FORMAT', 'https://abc.io/vuln/{ecosystem}:{package}') - monkeypatch.setenv('SNYK_SIGNIN_URL', 'https://abc.io/login') resp = StackAggregator().execute(_request_body(), persist=False) _mock_license.assert_called_once() _mock_gremlin.assert_called() @@ -99,14 +97,12 @@ def test_with_2_public_vuln(_mock_license, _mock_gremlin, monkeypatch): # check analyzed_dependencies result = StackAggregatorResult(**result) - assert result.registration_link == 'https://abc.io/login' assert len(result.analyzed_dependencies) == 2 assert _FLASK in result.analyzed_dependencies assert _SIX not in result.analyzed_dependencies # check vuln django_index = result.analyzed_dependencies.index(_DJANGO) - assert result.analyzed_dependencies[django_index].url == 'https://abc.io/vuln/pip:django' assert len(result.analyzed_dependencies[django_index].public_vulnerabilities) == 2 assert len(result.analyzed_dependencies[django_index].private_vulnerabilities) == 0 assert isinstance(result.analyzed_dependencies[django_index].public_vulnerabilities[0],