fix: Use ingestion API for unknown package flow

openshift/template.yaml

@@ -101,8 +101,6 @@ objects:
             value: "metrics-accumulator"
           - name: METRICS_ENDPOINT_URL_PORT
             value: "5200"
-          - name: DISABLE_UNKNOWN_PACKAGE_FLOW
-            value: ${DISABLE_UNKNOWN_PACKAGE_FLOW}
 
           image: "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${IMAGE_TAG}"
           name: f8a-server-backbone
@@ -228,10 +226,3 @@ parameters:
   required: false
   name: FLASK_LOGGING_LEVEL
   value: "INFO"
-
-- description: Flag to enable or disable the unknown package ingetion flow
-  displayName: DISABLE UNKNOWN PACKAGE FLOW
-  required: false
-  name: DISABLE_UNKNOWN_PACKAGE_FLOW
-  value: "0"
-

requirements.in

@@ -9,5 +9,5 @@ pydantic
 psycopg2-binary
 sqlalchemy
 raven[flask]
-f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker
-f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils
+f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker
+f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils

requirements.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements.txt requirements.in
 #
-amqp==5.0.2               # via kombu
+amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
 anymarkup==0.8.1          # via f8a-worker
 attrs==20.3.0             # via jsonschema
@@ -14,20 +14,19 @@ blinker==1.4              # via raven
 boto3==1.16.25            # via f8a-worker
 botocore==1.19.25         # via boto3, f8a-worker, s3transfer
 bs4==0.0.1                # via f8a-utils
-celery==5.0.2             # via selinon
+celery==4.4.7             # via f8a-worker
 certifi==2020.11.8        # via requests
 cffi==1.14.4              # via cryptography
 chardet==3.0.4            # via requests
-click-didyoumean==0.0.3   # via celery
-click-repl==0.1.6         # via celery
-click==7.1.2              # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon
+click==7.1.2              # via anymarkup, flask, selinon
 codegen==1.0              # via selinon
 colorama==0.4.4           # via rainbow-logging-handler
 configobj==5.0.6          # via anymarkup
 cryptography==3.2.1       # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils  # via -r requirements.in, f8a-worker
+dataclasses==0.8          # via pydantic
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via -r requirements.in, f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker  # via -r requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker  # via -r requirements.in
 flask-cors==3.0.9         # via -r requirements.in
 flask==1.1.2              # via -r requirements.in, flask-cors, raven
 gevent==20.9.0            # via -r requirements.in
@@ -45,11 +44,10 @@ jmespath==0.10.0          # via boto3, botocore
 jsl==0.2.4                # via f8a-worker
 json5==0.9.5              # via anymarkup
 jsonschema==3.2.0         # via f8a-worker, selinon
-kombu==5.0.2              # via celery
+kombu==4.6.11             # via celery
 logutils==0.3.5           # via rainbow-logging-handler
 lxml==4.6.2               # via f8a-utils, f8a-worker
 markupsafe==1.1.1         # via jinja2
-prompt-toolkit==3.0.8     # via click-repl
 psycopg2-binary==2.8.6    # via -r requirements.in
 pycparser==2.20           # via cffi
 pydantic==1.7.2           # via -r requirements.in
@@ -59,21 +57,20 @@ pytz==2020.4              # via celery
 pyyaml==5.3.1             # via anymarkup, f8a-worker, selinon
 rainbow-logging-handler==2.2.2  # via selinon
 raven[flask]==6.10.0      # via -r requirements.in, f8a-worker
-requests-futures==1.0.0   # via -r requirements.in, f8a-worker
+requests-futures==1.0.0   # via -r requirements.in, f8a-utils, f8a-worker
 requests==2.25.0          # via -r requirements.in, f8a-utils, f8a-worker, requests-futures
 s3transfer==0.3.3         # via boto3
-selinon[celery]==1.0.0    # via f8a-worker
+selinon==1.0.0            # via f8a-worker
 semantic-version==2.8.5   # via -r requirements.in, f8a-worker
 semver==2.13.0            # via f8a-utils
-six==1.15.0               # via anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity
+six==1.15.0               # via anymarkup-core, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity
 smmap==3.0.4              # via gitdb
 soupsieve==2.0.1          # via beautifulsoup4
 sqlalchemy==1.3.20        # via -r requirements.in, f8a-worker
 tenacity==6.2.0           # via f8a-utils, f8a-worker
 toml==0.9.4               # via anymarkup, f8a-worker
 urllib3==1.26.2           # via botocore, requests
-vine==5.0.0               # via amqp, celery
-wcwidth==0.2.5            # via prompt-toolkit
+vine==1.3.0               # via amqp, celery
 werkzeug==1.0.1           # via f8a-worker, flask
 xmltodict==0.12.0         # via anymarkup
 zipp==3.4.0               # via importlib-metadata

src/v2/stack_aggregator.py

@@ -13,9 +13,10 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
+from f8a_utils.ingestion_utils import unknown_package_flow
 
 from src.settings import AggregatorSettings
-from src.utils import (select_latest_version, server_create_analysis,
+from src.utils import (select_latest_version,
                        persist_data_in_db, post_gremlin, GREMLIN_QUERY_SIZE,
                        format_date)
 from src.v2.models import (StackAggregatorRequest, GitHubDetails, PackageDetails,
@@ -26,6 +27,7 @@
 from src.v2.normalized_packages import NormalizedPackages, GoNormalizedPackages
 from src.v2.license_service import (get_license_analysis_for_stack,
                                     get_license_service_request_payload)
+from f8a_utils import ingestion_utils
 
 logger = logging.getLogger(__name__)
 _TRUE = ['true', True, 1, '1']
@@ -306,19 +308,17 @@ def get_result(self) -> StackAggregatorResult:
 
     def initiate_unknown_package_ingestion(self):
         """Ingestion of Unknown dependencies."""
-        if AggregatorSettings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
-            return
-
         ecosystem = self._normalized_packages.ecosystem
+        pkg_list = self.get_all_unknown_packages()
+        unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
+                                                                   version=pkg.version), pkg_list))
         try:
-            for dep in self.get_all_unknown_packages():
-                server_create_analysis(ecosystem, dep.name, dep.version, api_flow=True,
-                                       force=False, force_graph_sync=True)
-        except Exception as e:  # pylint:disable=W0703,C0103
-            logger.error('Ingestion failed for {%s, %s, %s}',
-                         ecosystem, dep.name, dep.version)
-            logger.error(e)
+            unknown_package_flow(ecosystem, unknown_pkgs)
+        except Exception as e:
+            logger.error('Unknown ingestion failed with %s', e)
+        else:
+            logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
+                         len(pkg_list), ecosystem)
 
 
 class StackAggregator:
@@ -383,10 +383,17 @@ def __init__(self, request: StackAggregatorRequest = None,
 
     def initiate_unknown_package_ingestion(self):
         """Ingestion of Unknown dependencies."""
-        if AggregatorSettings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
-            return
-        logger.error('Ingestion is Not active for Golang.')
+        ecosystem = self._normalized_packages.ecosystem
+        pkg_list = self.get_all_unknown_packages()
+        unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
+                                                                   version=pkg.version), pkg_list))
+        try:
+            unknown_package_flow(ecosystem, unknown_pkgs)
+        except Exception as e:
+            logger.error('Unknown ingestion failed with %s', e)
+        else:
+            logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
+                         len(pkg_list), ecosystem)
 
     def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]:
         """Get package data from graph along with vulnerability."""

tests/requirements.txt

@@ -2,9 +2,9 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=tests/requirements.txt tests/requirements.in
+#    pip-compile --output-file=requirements.txt requirements.in
 #
-amqp==5.0.2               # via kombu
+amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
 anymarkup==0.8.1          # via f8a-worker
 attrs==20.3.0             # via jsonschema, pytest
@@ -14,32 +14,31 @@ blinker==1.4              # via raven
 boto3==1.16.25            # via f8a-worker
 botocore==1.19.25         # via boto3, f8a-worker, s3transfer
 bs4==0.0.1                # via f8a-utils
-celery==5.0.2             # via selinon
+celery==4.4.7             # via f8a-worker
 certifi==2020.11.8        # via requests
 cffi==1.14.4              # via cryptography
 chardet==3.0.4            # via requests
-click-didyoumean==0.0.3   # via celery
-click-repl==0.1.6         # via celery
-click==7.1.2              # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon
-codecov==2.1.10           # via -r tests/requirements.in
+click==7.1.2              # via anymarkup, flask, selinon
+codecov==2.1.10           # via -r requirements.in
 codegen==1.0              # via selinon
 colorama==0.4.4           # via radon, rainbow-logging-handler
 configobj==5.0.6          # via anymarkup
 coverage==5.3             # via codecov, pytest-cov
 cryptography==3.2.1       # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils  # via -r tests/../requirements.in, f8a-worker
+dataclasses==0.8          # via pydantic
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via -r ../requirements.in, f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker  # via -r tests/../requirements.in
-flask-cors==3.0.9         # via -r tests/../requirements.in
-flask==1.1.2              # via -r tests/../requirements.in, flask-cors, raven
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker  # via -r ../requirements.in
+flask-cors==3.0.9         # via -r ../requirements.in
+flask==1.1.2              # via -r ../requirements.in, flask-cors, raven
 future==0.18.2            # via radon
-gevent==20.9.0            # via -r tests/../requirements.in
+gevent==20.9.0            # via -r ../requirements.in
 git2json==0.2.3           # via f8a-worker
 gitdb==4.0.5              # via gitpython
 gitpython==3.1.11         # via f8a-worker
 graphviz==0.15            # via selinon
 greenlet==0.4.17          # via gevent
-gunicorn==20.0.4          # via -r tests/../requirements.in
+gunicorn==20.0.4          # via -r ../requirements.in
 idna==2.10                # via requests
 importlib-metadata==3.1.0  # via jsonschema, kombu, pluggy, pytest
 iniconfig==1.1.1          # via pytest
@@ -49,43 +48,41 @@ jmespath==0.10.0          # via boto3, botocore
 jsl==0.2.4                # via f8a-worker
 json5==0.9.5              # via anymarkup
 jsonschema==3.2.0         # via f8a-worker, selinon
-kombu==5.0.2              # via celery
+kombu==4.6.11             # via celery
 logutils==0.3.5           # via rainbow-logging-handler
 lxml==4.6.2               # via f8a-utils, f8a-worker
 mando==0.6.4              # via radon
 markupsafe==1.1.1         # via jinja2
 packaging==20.5           # via pytest
 pluggy==0.13.1            # via pytest
-prompt-toolkit==3.0.8     # via click-repl
-psycopg2-binary==2.8.6    # via -r tests/../requirements.in
+psycopg2-binary==2.8.6    # via -r ../requirements.in
 py==1.9.0                 # via pytest
 pycparser==2.20           # via cffi
-pydantic==1.7.2           # via -r tests/../requirements.in
+pydantic==1.7.2           # via -r ../requirements.in
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via jsonschema
-pytest-cov==2.10.1        # via -r tests/requirements.in
-pytest==6.1.2             # via -r tests/requirements.in, pytest-cov
+pytest-cov==2.10.1        # via -r requirements.in
+pytest==6.1.2             # via -r requirements.in, pytest-cov
 python-dateutil==2.8.1    # via botocore
 pytz==2020.4              # via celery
 pyyaml==5.3.1             # via anymarkup, f8a-worker, selinon
-radon==4.3.2              # via -r tests/requirements.in
+radon==4.3.2              # via -r requirements.in
 rainbow-logging-handler==2.2.2  # via selinon
-raven[flask]==6.10.0      # via -r tests/../requirements.in, f8a-worker
-requests-futures==1.0.0   # via -r tests/../requirements.in, f8a-worker
-requests==2.25.0          # via -r tests/../requirements.in, codecov, f8a-utils, f8a-worker, requests-futures
+raven[flask]==6.10.0      # via -r ../requirements.in, f8a-worker
+requests-futures==1.0.0   # via -r ../requirements.in, f8a-utils, f8a-worker
+requests==2.25.0          # via -r ../requirements.in, codecov, f8a-utils, f8a-worker, requests-futures
 s3transfer==0.3.3         # via boto3
-selinon[celery]==1.0.0    # via f8a-worker
-semantic-version==2.8.5   # via -r tests/../requirements.in, f8a-worker
+selinon==1.0.0            # via f8a-worker
+semantic-version==2.8.5   # via -r ../requirements.in, f8a-worker
 semver==2.13.0            # via f8a-utils
-six==1.15.0               # via anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, mando, python-dateutil, tenacity
+six==1.15.0               # via anymarkup-core, configobj, cryptography, flask-cors, jsonschema, mando, python-dateutil, tenacity
 smmap==3.0.4              # via gitdb
 soupsieve==2.0.1          # via beautifulsoup4
-sqlalchemy==1.3.20        # via -r tests/../requirements.in, f8a-worker
+sqlalchemy==1.3.20        # via -r ../requirements.in, f8a-worker
 tenacity==6.2.0           # via f8a-utils, f8a-worker
 toml==0.9.4               # via anymarkup, f8a-worker, pytest
 urllib3==1.26.2           # via botocore, requests
-vine==5.0.0               # via amqp, celery
-wcwidth==0.2.5            # via prompt-toolkit
+vine==1.3.0               # via amqp, celery
 werkzeug==1.0.1           # via f8a-worker, flask
 xmltodict==0.12.0         # via anymarkup
 zipp==3.4.0               # via importlib-metadata

tests/v2/test_stack_aggregator.py

@@ -203,24 +203,7 @@ def test_with_2_public_vuln_for_registered(_mock_license, _mock_gremlin):
                vulnerable_dependencies[0].public_vulnerabilities) == 2
 
 
-@mock.patch('src.v2.stack_aggregator.server_create_analysis')
-@mock.patch('src.v2.stack_aggregator.post_gremlin')
-def test_unknown_flow_with_disabled_flag(_mock_gremlin, _mock_unknown, monkeypatch):
-    """Test unknown flow."""
-    with open("tests/v2/data/graph_response_2_public_vuln.json", "r") as fin:
-        _mock_gremlin.return_value = json.load(fin)
-
-    payload = _request_body()
-    # add unknown package as direct dependency
-    payload['packages'].append(_SIX.dict())
-
-    # Disabled unknown flow check
-    monkeypatch.setenv('DISABLE_UNKNOWN_PACKAGE_FLOW', '1')
-    StackAggregator().execute(payload, persist=False)
-    _mock_unknown.assert_not_called()
-
-
-@mock.patch('src.v2.stack_aggregator.server_create_analysis')
+@mock.patch('src.v2.stack_aggregator.unknown_package_flow')
 @mock.patch('src.v2.stack_aggregator.post_gremlin')
 @mock.patch('src.v2.stack_aggregator.get_license_analysis_for_stack')
 def test_unknown_flow(_mock_license, _mock_gremlin, _mock_unknown):