diff --git a/openshift/template.yaml b/openshift/template.yaml index b2f525a6..dfef102c 100644 --- a/openshift/template.yaml +++ b/openshift/template.yaml @@ -101,8 +101,6 @@ objects: value: "metrics-accumulator" - name: METRICS_ENDPOINT_URL_PORT value: "5200" - - name: DISABLE_UNKNOWN_PACKAGE_FLOW - value: ${DISABLE_UNKNOWN_PACKAGE_FLOW} image: "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${IMAGE_TAG}" name: f8a-server-backbone @@ -228,10 +226,3 @@ parameters: required: false name: FLASK_LOGGING_LEVEL value: "INFO" - -- description: Flag to enable or disable the unknown package ingetion flow - displayName: DISABLE UNKNOWN PACKAGE FLOW - required: false - name: DISABLE_UNKNOWN_PACKAGE_FLOW - value: "0" - diff --git a/requirements.in b/requirements.in index 237a343d..066fae3f 100644 --- a/requirements.in +++ b/requirements.in @@ -9,5 +9,5 @@ pydantic psycopg2-binary sqlalchemy raven[flask] -f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker -f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils +f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker +f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils diff --git a/requirements.txt b/requirements.txt index 0a593040..255f36c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements.txt requirements.in # -amqp==5.0.2 # via kombu +amqp==2.6.1 # via kombu anymarkup-core==0.8.1 # via anymarkup anymarkup==0.8.1 # via f8a-worker attrs==20.3.0 # via jsonschema @@ -14,20 +14,19 @@ blinker==1.4 # via raven boto3==1.16.25 # via f8a-worker botocore==1.19.25 # via boto3, f8a-worker, s3transfer bs4==0.0.1 # via f8a-utils -celery==5.0.2 # via selinon +celery==4.4.7 # via f8a-worker certifi==2020.11.8 # via requests cffi==1.14.4 # via cryptography chardet==3.0.4 # via requests -click-didyoumean==0.0.3 # via celery -click-repl==0.1.6 # via celery -click==7.1.2 # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon +click==7.1.2 # via anymarkup, flask, selinon codegen==1.0 # via selinon colorama==0.4.4 # via rainbow-logging-handler configobj==5.0.6 # via anymarkup cryptography==3.2.1 # via f8a-utils -git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils # via -r requirements.in, f8a-worker +dataclasses==0.8 # via pydantic +git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils # via -r requirements.in, f8a-worker git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator # via f8a-utils -git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker # via -r requirements.in +git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker # via -r requirements.in flask-cors==3.0.9 # via -r requirements.in flask==1.1.2 # via -r requirements.in, flask-cors, raven gevent==20.9.0 # via -r requirements.in @@ -45,11 +44,10 @@ jmespath==0.10.0 # via boto3, botocore jsl==0.2.4 # via f8a-worker json5==0.9.5 # via anymarkup jsonschema==3.2.0 # via f8a-worker, selinon -kombu==5.0.2 # via celery +kombu==4.6.11 # via celery logutils==0.3.5 # via rainbow-logging-handler lxml==4.6.2 # via f8a-utils, f8a-worker markupsafe==1.1.1 # via jinja2 -prompt-toolkit==3.0.8 # via click-repl psycopg2-binary==2.8.6 # via -r requirements.in pycparser==2.20 # via cffi pydantic==1.7.2 # via -r requirements.in @@ -59,21 +57,20 @@ pytz==2020.4 # via celery pyyaml==5.3.1 # via anymarkup, f8a-worker, selinon rainbow-logging-handler==2.2.2 # via selinon raven[flask]==6.10.0 # via -r requirements.in, f8a-worker -requests-futures==1.0.0 # via -r requirements.in, f8a-worker +requests-futures==1.0.0 # via -r requirements.in, f8a-utils, f8a-worker requests==2.25.0 # via -r requirements.in, f8a-utils, f8a-worker, requests-futures s3transfer==0.3.3 # via boto3 -selinon[celery]==1.0.0 # via f8a-worker +selinon==1.0.0 # via f8a-worker semantic-version==2.8.5 # via -r requirements.in, f8a-worker semver==2.13.0 # via f8a-utils -six==1.15.0 # via anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity +six==1.15.0 # via anymarkup-core, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity smmap==3.0.4 # via gitdb soupsieve==2.0.1 # via beautifulsoup4 sqlalchemy==1.3.20 # via -r requirements.in, f8a-worker tenacity==6.2.0 # via f8a-utils, f8a-worker toml==0.9.4 # via anymarkup, f8a-worker urllib3==1.26.2 # via botocore, requests -vine==5.0.0 # via amqp, celery -wcwidth==0.2.5 # via prompt-toolkit +vine==1.3.0 # via amqp, celery werkzeug==1.0.1 # via f8a-worker, flask xmltodict==0.12.0 # via anymarkup zipp==3.4.0 # via importlib-metadata diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py index 9d182cf7..b976e767 100644 --- a/src/v2/stack_aggregator.py +++ b/src/v2/stack_aggregator.py @@ -13,9 +13,10 @@ from typing import Dict, List, Tuple, Set from f8a_utils.gh_utils import GithubUtils +from f8a_utils.ingestion_utils import unknown_package_flow from src.settings import AggregatorSettings -from src.utils import (select_latest_version, server_create_analysis, +from src.utils import (select_latest_version, persist_data_in_db, post_gremlin, GREMLIN_QUERY_SIZE, format_date) from src.v2.models import (StackAggregatorRequest, GitHubDetails, PackageDetails, @@ -26,6 +27,7 @@ from src.v2.normalized_packages import NormalizedPackages, GoNormalizedPackages from src.v2.license_service import (get_license_analysis_for_stack, get_license_service_request_payload) +from f8a_utils import ingestion_utils logger = logging.getLogger(__name__) _TRUE = ['true', True, 1, '1'] @@ -306,19 +308,17 @@ def get_result(self) -> StackAggregatorResult: def initiate_unknown_package_ingestion(self): """Ingestion of Unknown dependencies.""" - if AggregatorSettings().disable_unknown_package_flow: - logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages()) - return - ecosystem = self._normalized_packages.ecosystem + pkg_list = self.get_all_unknown_packages() + unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name, + version=pkg.version), pkg_list)) try: - for dep in self.get_all_unknown_packages(): - server_create_analysis(ecosystem, dep.name, dep.version, api_flow=True, - force=False, force_graph_sync=True) - except Exception as e: # pylint:disable=W0703,C0103 - logger.error('Ingestion failed for {%s, %s, %s}', - ecosystem, dep.name, dep.version) - logger.error(e) + unknown_package_flow(ecosystem, unknown_pkgs) + except Exception as e: + logger.error('Unknown ingestion failed with %s', e) + else: + logger.debug('Unknown ingestion executed for %s packages in %s ecosystem', + len(pkg_list), ecosystem) class StackAggregator: @@ -383,10 +383,17 @@ def __init__(self, request: StackAggregatorRequest = None, def initiate_unknown_package_ingestion(self): """Ingestion of Unknown dependencies.""" - if AggregatorSettings().disable_unknown_package_flow: - logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages()) - return - logger.error('Ingestion is Not active for Golang.') + ecosystem = self._normalized_packages.ecosystem + pkg_list = self.get_all_unknown_packages() + unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name, + version=pkg.version), pkg_list)) + try: + unknown_package_flow(ecosystem, unknown_pkgs) + except Exception as e: + logger.error('Unknown ingestion failed with %s', e) + else: + logger.debug('Unknown ingestion executed for %s packages in %s ecosystem', + len(pkg_list), ecosystem) def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]: """Get package data from graph along with vulnerability.""" diff --git a/tests/requirements.txt b/tests/requirements.txt index bb29850f..76386b3a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,9 +2,9 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file=tests/requirements.txt tests/requirements.in +# pip-compile --output-file=requirements.txt requirements.in # -amqp==5.0.2 # via kombu +amqp==2.6.1 # via kombu anymarkup-core==0.8.1 # via anymarkup anymarkup==0.8.1 # via f8a-worker attrs==20.3.0 # via jsonschema, pytest @@ -14,32 +14,31 @@ blinker==1.4 # via raven boto3==1.16.25 # via f8a-worker botocore==1.19.25 # via boto3, f8a-worker, s3transfer bs4==0.0.1 # via f8a-utils -celery==5.0.2 # via selinon +celery==4.4.7 # via f8a-worker certifi==2020.11.8 # via requests cffi==1.14.4 # via cryptography chardet==3.0.4 # via requests -click-didyoumean==0.0.3 # via celery -click-repl==0.1.6 # via celery -click==7.1.2 # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon -codecov==2.1.10 # via -r tests/requirements.in +click==7.1.2 # via anymarkup, flask, selinon +codecov==2.1.10 # via -r requirements.in codegen==1.0 # via selinon colorama==0.4.4 # via radon, rainbow-logging-handler configobj==5.0.6 # via anymarkup coverage==5.3 # via codecov, pytest-cov cryptography==3.2.1 # via f8a-utils -git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils # via -r tests/../requirements.in, f8a-worker +dataclasses==0.8 # via pydantic +git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils # via -r ../requirements.in, f8a-worker git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator # via f8a-utils -git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker # via -r tests/../requirements.in -flask-cors==3.0.9 # via -r tests/../requirements.in -flask==1.1.2 # via -r tests/../requirements.in, flask-cors, raven +git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker # via -r ../requirements.in +flask-cors==3.0.9 # via -r ../requirements.in +flask==1.1.2 # via -r ../requirements.in, flask-cors, raven future==0.18.2 # via radon -gevent==20.9.0 # via -r tests/../requirements.in +gevent==20.9.0 # via -r ../requirements.in git2json==0.2.3 # via f8a-worker gitdb==4.0.5 # via gitpython gitpython==3.1.11 # via f8a-worker graphviz==0.15 # via selinon greenlet==0.4.17 # via gevent -gunicorn==20.0.4 # via -r tests/../requirements.in +gunicorn==20.0.4 # via -r ../requirements.in idna==2.10 # via requests importlib-metadata==3.1.0 # via jsonschema, kombu, pluggy, pytest iniconfig==1.1.1 # via pytest @@ -49,43 +48,41 @@ jmespath==0.10.0 # via boto3, botocore jsl==0.2.4 # via f8a-worker json5==0.9.5 # via anymarkup jsonschema==3.2.0 # via f8a-worker, selinon -kombu==5.0.2 # via celery +kombu==4.6.11 # via celery logutils==0.3.5 # via rainbow-logging-handler lxml==4.6.2 # via f8a-utils, f8a-worker mando==0.6.4 # via radon markupsafe==1.1.1 # via jinja2 packaging==20.5 # via pytest pluggy==0.13.1 # via pytest -prompt-toolkit==3.0.8 # via click-repl -psycopg2-binary==2.8.6 # via -r tests/../requirements.in +psycopg2-binary==2.8.6 # via -r ../requirements.in py==1.9.0 # via pytest pycparser==2.20 # via cffi -pydantic==1.7.2 # via -r tests/../requirements.in +pydantic==1.7.2 # via -r ../requirements.in pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -pytest-cov==2.10.1 # via -r tests/requirements.in -pytest==6.1.2 # via -r tests/requirements.in, pytest-cov +pytest-cov==2.10.1 # via -r requirements.in +pytest==6.1.2 # via -r requirements.in, pytest-cov python-dateutil==2.8.1 # via botocore pytz==2020.4 # via celery pyyaml==5.3.1 # via anymarkup, f8a-worker, selinon -radon==4.3.2 # via -r tests/requirements.in +radon==4.3.2 # via -r requirements.in rainbow-logging-handler==2.2.2 # via selinon -raven[flask]==6.10.0 # via -r tests/../requirements.in, f8a-worker -requests-futures==1.0.0 # via -r tests/../requirements.in, f8a-worker -requests==2.25.0 # via -r tests/../requirements.in, codecov, f8a-utils, f8a-worker, requests-futures +raven[flask]==6.10.0 # via -r ../requirements.in, f8a-worker +requests-futures==1.0.0 # via -r ../requirements.in, f8a-utils, f8a-worker +requests==2.25.0 # via -r ../requirements.in, codecov, f8a-utils, f8a-worker, requests-futures s3transfer==0.3.3 # via boto3 -selinon[celery]==1.0.0 # via f8a-worker -semantic-version==2.8.5 # via -r tests/../requirements.in, f8a-worker +selinon==1.0.0 # via f8a-worker +semantic-version==2.8.5 # via -r ../requirements.in, f8a-worker semver==2.13.0 # via f8a-utils -six==1.15.0 # via anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, mando, python-dateutil, tenacity +six==1.15.0 # via anymarkup-core, configobj, cryptography, flask-cors, jsonschema, mando, python-dateutil, tenacity smmap==3.0.4 # via gitdb soupsieve==2.0.1 # via beautifulsoup4 -sqlalchemy==1.3.20 # via -r tests/../requirements.in, f8a-worker +sqlalchemy==1.3.20 # via -r ../requirements.in, f8a-worker tenacity==6.2.0 # via f8a-utils, f8a-worker toml==0.9.4 # via anymarkup, f8a-worker, pytest urllib3==1.26.2 # via botocore, requests -vine==5.0.0 # via amqp, celery -wcwidth==0.2.5 # via prompt-toolkit +vine==1.3.0 # via amqp, celery werkzeug==1.0.1 # via f8a-worker, flask xmltodict==0.12.0 # via anymarkup zipp==3.4.0 # via importlib-metadata diff --git a/tests/v2/test_stack_aggregator.py b/tests/v2/test_stack_aggregator.py index 6890c7e6..3b61f21c 100644 --- a/tests/v2/test_stack_aggregator.py +++ b/tests/v2/test_stack_aggregator.py @@ -203,24 +203,7 @@ def test_with_2_public_vuln_for_registered(_mock_license, _mock_gremlin): vulnerable_dependencies[0].public_vulnerabilities) == 2 -@mock.patch('src.v2.stack_aggregator.server_create_analysis') -@mock.patch('src.v2.stack_aggregator.post_gremlin') -def test_unknown_flow_with_disabled_flag(_mock_gremlin, _mock_unknown, monkeypatch): - """Test unknown flow.""" - with open("tests/v2/data/graph_response_2_public_vuln.json", "r") as fin: - _mock_gremlin.return_value = json.load(fin) - - payload = _request_body() - # add unknown package as direct dependency - payload['packages'].append(_SIX.dict()) - - # Disabled unknown flow check - monkeypatch.setenv('DISABLE_UNKNOWN_PACKAGE_FLOW', '1') - StackAggregator().execute(payload, persist=False) - _mock_unknown.assert_not_called() - - -@mock.patch('src.v2.stack_aggregator.server_create_analysis') +@mock.patch('src.v2.stack_aggregator.unknown_package_flow') @mock.patch('src.v2.stack_aggregator.post_gremlin') @mock.patch('src.v2.stack_aggregator.get_license_analysis_for_stack') def test_unknown_flow(_mock_license, _mock_gremlin, _mock_unknown):