From 7b129fb33b828d5e11c0792991915aa8d4edac4e Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Mon, 4 Jan 2021 17:31:25 +0530
Subject: [PATCH 1/8] fix: [SA] Trigger unknown package using Ingestion API.

---
 requirements.in                   |  3 +-
 requirements.txt                  | 27 +++++++--------
 src/v2/stack_aggregator.py        | 38 ++++++++++++++++-----
 tests/requirements.txt            | 55 +++++++++++++++----------------
 tests/v2/test_stack_aggregator.py |  4 +--
 5 files changed, 71 insertions(+), 56 deletions(-)

diff --git a/requirements.in b/requirements.in
index 237a343d..065409bb 100644
--- a/requirements.in
+++ b/requirements.in
@@ -9,5 +9,4 @@ pydantic
 psycopg2-binary
 sqlalchemy
 raven[flask]
-f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker
-f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils
+f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@b4e1fea#egg=f8a_worker
diff --git a/requirements.txt b/requirements.txt
index 0a593040..e89bd098 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,9 +2,9 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=requirements.txt requirements.in
+#    pip-compile
 #
-amqp==5.0.2               # via kombu
+amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
 anymarkup==0.8.1          # via f8a-worker
 attrs==20.3.0             # via jsonschema
@@ -14,20 +14,19 @@ blinker==1.4              # via raven
 boto3==1.16.25            # via f8a-worker
 botocore==1.19.25         # via boto3, f8a-worker, s3transfer
 bs4==0.0.1                # via f8a-utils
-celery==5.0.2             # via selinon
+celery==4.4.7             # via f8a-worker
 certifi==2020.11.8        # via requests
 cffi==1.14.4              # via cryptography
 chardet==3.0.4            # via requests
-click-didyoumean==0.0.3   # via celery
-click-repl==0.1.6         # via celery
-click==7.1.2              # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon
+click==7.1.2              # via anymarkup, flask, selinon
 codegen==1.0              # via selinon
 colorama==0.4.4           # via rainbow-logging-handler
 configobj==5.0.6          # via anymarkup
 cryptography==3.2.1       # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils  # via -r requirements.in, f8a-worker
+dataclasses==0.8          # via pydantic
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@64e83a7#egg=f8a_utils  # via f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker  # via -r requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@b4e1fea#egg=f8a_worker  # via -r requirements.in
 flask-cors==3.0.9         # via -r requirements.in
 flask==1.1.2              # via -r requirements.in, flask-cors, raven
 gevent==20.9.0            # via -r requirements.in
@@ -45,11 +44,10 @@ jmespath==0.10.0          # via boto3, botocore
 jsl==0.2.4                # via f8a-worker
 json5==0.9.5              # via anymarkup
 jsonschema==3.2.0         # via f8a-worker, selinon
-kombu==5.0.2              # via celery
+kombu==4.6.11             # via celery
 logutils==0.3.5           # via rainbow-logging-handler
 lxml==4.6.2               # via f8a-utils, f8a-worker
 markupsafe==1.1.1         # via jinja2
-prompt-toolkit==3.0.8     # via click-repl
 psycopg2-binary==2.8.6    # via -r requirements.in
 pycparser==2.20           # via cffi
 pydantic==1.7.2           # via -r requirements.in
@@ -59,21 +57,20 @@ pytz==2020.4              # via celery
 pyyaml==5.3.1             # via anymarkup, f8a-worker, selinon
 rainbow-logging-handler==2.2.2  # via selinon
 raven[flask]==6.10.0      # via -r requirements.in, f8a-worker
-requests-futures==1.0.0   # via -r requirements.in, f8a-worker
+requests-futures==1.0.0   # via -r requirements.in, f8a-utils, f8a-worker
 requests==2.25.0          # via -r requirements.in, f8a-utils, f8a-worker, requests-futures
 s3transfer==0.3.3         # via boto3
-selinon[celery]==1.0.0    # via f8a-worker
+selinon==1.0.0            # via f8a-worker
 semantic-version==2.8.5   # via -r requirements.in, f8a-worker
 semver==2.13.0            # via f8a-utils
-six==1.15.0               # via anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity
+six==1.15.0               # via anymarkup-core, configobj, cryptography, flask-cors, jsonschema, python-dateutil, tenacity
 smmap==3.0.4              # via gitdb
 soupsieve==2.0.1          # via beautifulsoup4
 sqlalchemy==1.3.20        # via -r requirements.in, f8a-worker
 tenacity==6.2.0           # via f8a-utils, f8a-worker
 toml==0.9.4               # via anymarkup, f8a-worker
 urllib3==1.26.2           # via botocore, requests
-vine==5.0.0               # via amqp, celery
-wcwidth==0.2.5            # via prompt-toolkit
+vine==1.3.0               # via amqp, celery
 werkzeug==1.0.1           # via f8a-worker, flask
 xmltodict==0.12.0         # via anymarkup
 zipp==3.4.0               # via importlib-metadata
diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index e74c653b..8c5be04f 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -13,9 +13,10 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
+from f8a_utils.ingestion_utils import unknown_package_flow
 
 from src.settings import Settings
-from src.utils import (select_latest_version, server_create_analysis,
+from src.utils import (select_latest_version,
                        persist_data_in_db, post_gremlin, GREMLIN_QUERY_SIZE,
                        format_date)
 from src.v2.models import (StackAggregatorRequest, GitHubDetails, PackageDetails,
@@ -26,9 +27,11 @@
 from src.v2.normalized_packages import NormalizedPackages, GoNormalizedPackages
 from src.v2.license_service import (get_license_analysis_for_stack,
                                     get_license_service_request_payload)
+from collections import namedtuple
 
 logger = logging.getLogger(__name__)
 _TRUE = ['true', True, 1, '1']
+_PACKAGE = namedtuple("Package", ["package", "version"])
 
 
 def _is_private_vulnerability(vulnerability_node):
@@ -309,13 +312,18 @@ def initiate_unknown_package_ingestion(self):
             return
 
         ecosystem = self._normalized_packages.ecosystem
+        pkg_list = self.get_all_unknown_packages()
         try:
-            for dep in self.get_all_unknown_packages():
-                server_create_analysis(ecosystem, dep.name, dep.version, api_flow=True,
-                                       force=False, force_graph_sync=True)
-        except Exception as e:  # pylint:disable=W0703,C0103
-            logger.error('Ingestion failed for {%s, %s, %s}',
-                         ecosystem, dep.name, dep.version)
+            if pkg_list:
+                # Converting tuple keys as utils expects different key names
+                pkg_list_temp = set()
+                for pkg in pkg_list:
+                    pkg_list_temp.add(_PACKAGE(package=pkg.name, version=pkg.version))
+
+                logger.info('Initializing unknown pakage ingestion for eco {} and Packages {}.'
+                            .format(ecosystem, pkg_list))
+                unknown_package_flow(ecosystem, pkg_list_temp)
+        except Exception as e:
             logger.error(e)
 
 
@@ -384,7 +392,21 @@ def initiate_unknown_package_ingestion(self):
         if Settings().disable_unknown_package_flow:
             logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
             return
-        logger.error('Ingestion is Not active for Golang.')
+
+        ecosystem = self._normalized_packages.ecosystem
+        pkg_list = self.get_all_unknown_packages()
+        try:
+            if pkg_list:
+                # Converting tuple keys as utils expects different key names
+                pkg_list_temp = set()
+                for pkg in pkg_list:
+                    pkg_list_temp.add(_PACKAGE(package=pkg.name, version=pkg.version))
+
+                logger.info('Initializing unknown pakage ingestion for eco {} and Packages {}.'
+                            .format(ecosystem, pkg_list_temp))
+                unknown_package_flow(ecosystem, pkg_list_temp)
+        except Exception as e:
+            logger.error(e)
 
     def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]:
         """Get package data from graph along with vulnerability."""
diff --git a/tests/requirements.txt b/tests/requirements.txt
index bb29850f..4860230a 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,9 +2,9 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=tests/requirements.txt tests/requirements.in
+#    pip-compile
 #
-amqp==5.0.2               # via kombu
+amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
 anymarkup==0.8.1          # via f8a-worker
 attrs==20.3.0             # via jsonschema, pytest
@@ -14,32 +14,31 @@ blinker==1.4              # via raven
 boto3==1.16.25            # via f8a-worker
 botocore==1.19.25         # via boto3, f8a-worker, s3transfer
 bs4==0.0.1                # via f8a-utils
-celery==5.0.2             # via selinon
+celery==4.4.7             # via f8a-worker
 certifi==2020.11.8        # via requests
 cffi==1.14.4              # via cryptography
 chardet==3.0.4            # via requests
-click-didyoumean==0.0.3   # via celery
-click-repl==0.1.6         # via celery
-click==7.1.2              # via anymarkup, celery, click-didyoumean, click-repl, flask, selinon
-codecov==2.1.10           # via -r tests/requirements.in
+click==7.1.2              # via anymarkup, flask, selinon
+codecov==2.1.10           # via -r requirements.in
 codegen==1.0              # via selinon
 colorama==0.4.4           # via radon, rainbow-logging-handler
 configobj==5.0.6          # via anymarkup
 coverage==5.3             # via codecov, pytest-cov
 cryptography==3.2.1       # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@5a5ce60#egg=f8a_utils  # via -r tests/../requirements.in, f8a-worker
+dataclasses==0.8          # via pydantic
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@64e83a7#egg=f8a_utils  # via f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@066c2f6#egg=f8a_worker  # via -r tests/../requirements.in
-flask-cors==3.0.9         # via -r tests/../requirements.in
-flask==1.1.2              # via -r tests/../requirements.in, flask-cors, raven
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@b4e1fea#egg=f8a_worker  # via -r ../requirements.in
+flask-cors==3.0.9         # via -r ../requirements.in
+flask==1.1.2              # via -r ../requirements.in, flask-cors, raven
 future==0.18.2            # via radon
-gevent==20.9.0            # via -r tests/../requirements.in
+gevent==20.9.0            # via -r ../requirements.in
 git2json==0.2.3           # via f8a-worker
 gitdb==4.0.5              # via gitpython
 gitpython==3.1.11         # via f8a-worker
 graphviz==0.15            # via selinon
 greenlet==0.4.17          # via gevent
-gunicorn==20.0.4          # via -r tests/../requirements.in
+gunicorn==20.0.4          # via -r ../requirements.in
 idna==2.10                # via requests
 importlib-metadata==3.1.0  # via jsonschema, kombu, pluggy, pytest
 iniconfig==1.1.1          # via pytest
@@ -49,43 +48,41 @@ jmespath==0.10.0          # via boto3, botocore
 jsl==0.2.4                # via f8a-worker
 json5==0.9.5              # via anymarkup
 jsonschema==3.2.0         # via f8a-worker, selinon
-kombu==5.0.2              # via celery
+kombu==4.6.11             # via celery
 logutils==0.3.5           # via rainbow-logging-handler
 lxml==4.6.2               # via f8a-utils, f8a-worker
 mando==0.6.4              # via radon
 markupsafe==1.1.1         # via jinja2
 packaging==20.5           # via pytest
 pluggy==0.13.1            # via pytest
-prompt-toolkit==3.0.8     # via click-repl
-psycopg2-binary==2.8.6    # via -r tests/../requirements.in
+psycopg2-binary==2.8.6    # via -r ../requirements.in
 py==1.9.0                 # via pytest
 pycparser==2.20           # via cffi
-pydantic==1.7.2           # via -r tests/../requirements.in
+pydantic==1.7.2           # via -r ../requirements.in
 pyparsing==2.4.7          # via packaging
 pyrsistent==0.17.3        # via jsonschema
-pytest-cov==2.10.1        # via -r tests/requirements.in
-pytest==6.1.2             # via -r tests/requirements.in, pytest-cov
+pytest-cov==2.10.1        # via -r requirements.in
+pytest==6.1.2             # via -r requirements.in, pytest-cov
 python-dateutil==2.8.1    # via botocore
 pytz==2020.4              # via celery
 pyyaml==5.3.1             # via anymarkup, f8a-worker, selinon
-radon==4.3.2              # via -r tests/requirements.in
+radon==4.3.2              # via -r requirements.in
 rainbow-logging-handler==2.2.2  # via selinon
-raven[flask]==6.10.0      # via -r tests/../requirements.in, f8a-worker
-requests-futures==1.0.0   # via -r tests/../requirements.in, f8a-worker
-requests==2.25.0          # via -r tests/../requirements.in, codecov, f8a-utils, f8a-worker, requests-futures
+raven[flask]==6.10.0      # via -r ../requirements.in, f8a-worker
+requests-futures==1.0.0   # via -r ../requirements.in, f8a-utils, f8a-worker
+requests==2.25.0          # via -r ../requirements.in, codecov, f8a-utils, f8a-worker, requests-futures
 s3transfer==0.3.3         # via boto3
-selinon[celery]==1.0.0    # via f8a-worker
-semantic-version==2.8.5   # via -r tests/../requirements.in, f8a-worker
+selinon==1.0.0            # via f8a-worker
+semantic-version==2.8.5   # via -r ../requirements.in, f8a-worker
 semver==2.13.0            # via f8a-utils
-six==1.15.0               # via anymarkup-core, click-repl, configobj, cryptography, flask-cors, jsonschema, mando, python-dateutil, tenacity
+six==1.15.0               # via anymarkup-core, configobj, cryptography, flask-cors, jsonschema, mando, python-dateutil, tenacity
 smmap==3.0.4              # via gitdb
 soupsieve==2.0.1          # via beautifulsoup4
-sqlalchemy==1.3.20        # via -r tests/../requirements.in, f8a-worker
+sqlalchemy==1.3.20        # via -r ../requirements.in, f8a-worker
 tenacity==6.2.0           # via f8a-utils, f8a-worker
 toml==0.9.4               # via anymarkup, f8a-worker, pytest
 urllib3==1.26.2           # via botocore, requests
-vine==5.0.0               # via amqp, celery
-wcwidth==0.2.5            # via prompt-toolkit
+vine==1.3.0               # via amqp, celery
 werkzeug==1.0.1           # via f8a-worker, flask
 xmltodict==0.12.0         # via anymarkup
 zipp==3.4.0               # via importlib-metadata
diff --git a/tests/v2/test_stack_aggregator.py b/tests/v2/test_stack_aggregator.py
index 6890c7e6..841e8260 100644
--- a/tests/v2/test_stack_aggregator.py
+++ b/tests/v2/test_stack_aggregator.py
@@ -203,7 +203,7 @@ def test_with_2_public_vuln_for_registered(_mock_license, _mock_gremlin):
                vulnerable_dependencies[0].public_vulnerabilities) == 2
 
 
-@mock.patch('src.v2.stack_aggregator.server_create_analysis')
+@mock.patch('src.v2.stack_aggregator.unknown_package_flow')
 @mock.patch('src.v2.stack_aggregator.post_gremlin')
 def test_unknown_flow_with_disabled_flag(_mock_gremlin, _mock_unknown, monkeypatch):
     """Test unknown flow."""
@@ -220,7 +220,7 @@ def test_unknown_flow_with_disabled_flag(_mock_gremlin, _mock_unknown, monkeypat
     _mock_unknown.assert_not_called()
 
 
-@mock.patch('src.v2.stack_aggregator.server_create_analysis')
+@mock.patch('src.v2.stack_aggregator.unknown_package_flow')
 @mock.patch('src.v2.stack_aggregator.post_gremlin')
 @mock.patch('src.v2.stack_aggregator.get_license_analysis_for_stack')
 def test_unknown_flow(_mock_license, _mock_gremlin, _mock_unknown):

From 8404908a51a30a6d34e2da015adad3ea402b816a Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Mon, 4 Jan 2021 17:55:51 +0530
Subject: [PATCH 2/8] pip-compile comment updated.

---
 requirements.txt       | 2 +-
 tests/requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index e89bd098..40e22b1d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 4860230a..8a0bfd52 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile
+#    pip-compile --output-file=tests/requirements.txt tests/requirements.in
 #
 amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup

From b2563e481059dc00184c2d89c52751baa299527f Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Mon, 4 Jan 2021 21:58:03 +0530
Subject: [PATCH 3/8] changed conversion mechanism.

---
 src/v2/stack_aggregator.py | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index 8c5be04f..abb0c6ec 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -314,15 +314,11 @@ def initiate_unknown_package_ingestion(self):
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
         try:
-            if pkg_list:
-                # Converting tuple keys as utils expects different key names
-                pkg_list_temp = set()
-                for pkg in pkg_list:
-                    pkg_list_temp.add(_PACKAGE(package=pkg.name, version=pkg.version))
-
-                logger.info('Initializing unknown pakage ingestion for eco {} and Packages {}.'
-                            .format(ecosystem, pkg_list))
-                unknown_package_flow(ecosystem, pkg_list_temp)
+            unknown_pkgs = set(map(lambda pkg: _PACKAGE(package=pkg.name,
+                                                        version=pkg.version), pkg_list))
+            unknown_package_flow(ecosystem, unknown_pkgs)
+            logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
+                         len(pkg_list), ecosystem)
         except Exception as e:
             logger.error(e)
 
@@ -396,15 +392,11 @@ def initiate_unknown_package_ingestion(self):
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
         try:
-            if pkg_list:
-                # Converting tuple keys as utils expects different key names
-                pkg_list_temp = set()
-                for pkg in pkg_list:
-                    pkg_list_temp.add(_PACKAGE(package=pkg.name, version=pkg.version))
-
-                logger.info('Initializing unknown pakage ingestion for eco {} and Packages {}.'
-                            .format(ecosystem, pkg_list_temp))
-                unknown_package_flow(ecosystem, pkg_list_temp)
+            unknown_pkgs = set(map(lambda pkg: _PACKAGE(package=pkg.name,
+                                                        version=pkg.version), pkg_list))
+            unknown_package_flow(ecosystem, unknown_pkgs)
+            logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
+                         len(pkg_list), ecosystem)
         except Exception as e:
             logger.error(e)
 

From 02fadfdad42103a536c726f92e35302ba84f8ebe Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Tue, 5 Jan 2021 14:17:12 +0530
Subject: [PATCH 4/8] updated dependency versions and used custom data type
 from utis.

---
 requirements.in            |  2 +-
 requirements.txt           |  4 ++--
 src/v2/stack_aggregator.py | 22 +++++++++++-----------
 tests/requirements.txt     |  6 +++---
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/requirements.in b/requirements.in
index 065409bb..b63c87eb 100644
--- a/requirements.in
+++ b/requirements.in
@@ -9,4 +9,4 @@ pydantic
 psycopg2-binary
 sqlalchemy
 raven[flask]
-f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@b4e1fea#egg=f8a_worker
+f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker
diff --git a/requirements.txt b/requirements.txt
index 40e22b1d..f8312638 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,9 +24,9 @@ colorama==0.4.4           # via rainbow-logging-handler
 configobj==5.0.6          # via anymarkup
 cryptography==3.2.1       # via f8a-utils
 dataclasses==0.8          # via pydantic
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@64e83a7#egg=f8a_utils  # via f8a-worker
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@b4e1fea#egg=f8a_worker  # via -r requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker  # via -r requirements.in
 flask-cors==3.0.9         # via -r requirements.in
 flask==1.1.2              # via -r requirements.in, flask-cors, raven
 gevent==20.9.0            # via -r requirements.in
diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index abb0c6ec..407d82f7 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -13,7 +13,7 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
-from f8a_utils.ingestion_utils import unknown_package_flow
+from f8a_utils.ingestion_utils import unknown_package_flow, Package
 
 from src.settings import Settings
 from src.utils import (select_latest_version,
@@ -27,11 +27,9 @@
 from src.v2.normalized_packages import NormalizedPackages, GoNormalizedPackages
 from src.v2.license_service import (get_license_analysis_for_stack,
                                     get_license_service_request_payload)
-from collections import namedtuple
 
 logger = logging.getLogger(__name__)
 _TRUE = ['true', True, 1, '1']
-_PACKAGE = namedtuple("Package", ["package", "version"])
 
 
 def _is_private_vulnerability(vulnerability_node):
@@ -313,14 +311,15 @@ def initiate_unknown_package_ingestion(self):
 
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
+        unknown_pkgs = set(map(lambda pkg: Package(package=pkg.name,
+                                                   version=pkg.version), pkg_list))
         try:
-            unknown_pkgs = set(map(lambda pkg: _PACKAGE(package=pkg.name,
-                                                        version=pkg.version), pkg_list))
             unknown_package_flow(ecosystem, unknown_pkgs)
+        except Exception as e:
+            logger.error('Unknown ingestion failed with %s', e)
+        else:
             logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
                          len(pkg_list), ecosystem)
-        except Exception as e:
-            logger.error(e)
 
 
 class StackAggregator:
@@ -391,14 +390,15 @@ def initiate_unknown_package_ingestion(self):
 
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
+        unknown_pkgs = set(map(lambda pkg: Package(package=pkg.name,
+                                                   version=pkg.version), pkg_list))
         try:
-            unknown_pkgs = set(map(lambda pkg: _PACKAGE(package=pkg.name,
-                                                        version=pkg.version), pkg_list))
             unknown_package_flow(ecosystem, unknown_pkgs)
+        except Exception as e:
+            logger.error('Unknown ingestion failed with %s', e)
+        else:
             logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
                          len(pkg_list), ecosystem)
-        except Exception as e:
-            logger.error(e)
 
     def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]:
         """Get package data from graph along with vulnerability."""
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 8a0bfd52..9d1dd2dd 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file=tests/requirements.txt tests/requirements.in
+#    pip-compile
 #
 amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
@@ -26,9 +26,9 @@ configobj==5.0.6          # via anymarkup
 coverage==5.3             # via codecov, pytest-cov
 cryptography==3.2.1       # via f8a-utils
 dataclasses==0.8          # via pydantic
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@64e83a7#egg=f8a_utils  # via f8a-worker
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
-git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@b4e1fea#egg=f8a_worker  # via -r ../requirements.in
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker  # via -r ../requirements.in
 flask-cors==3.0.9         # via -r ../requirements.in
 flask==1.1.2              # via -r ../requirements.in, flask-cors, raven
 future==0.18.2            # via radon

From 0c5af7f690c48bb76af4de33f033b84ef643c3eb Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Tue, 5 Jan 2021 15:21:09 +0530
Subject: [PATCH 5/8] PyLint Fix

---
 src/v2/stack_aggregator.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index 407d82f7..e9b44158 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -13,7 +13,8 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
-from f8a_utils.ingestion_utils import unknown_package_flow, Package
+from f8a_utils.ingestion_utils import unknown_package_flow, \
+    Package as P
 
 from src.settings import Settings
 from src.utils import (select_latest_version,
@@ -311,8 +312,8 @@ def initiate_unknown_package_ingestion(self):
 
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
-        unknown_pkgs = set(map(lambda pkg: Package(package=pkg.name,
-                                                   version=pkg.version), pkg_list))
+        unknown_pkgs = set(map(lambda pkg: P(package=pkg.name,
+                                             version=pkg.version), pkg_list))
         try:
             unknown_package_flow(ecosystem, unknown_pkgs)
         except Exception as e:
@@ -390,8 +391,8 @@ def initiate_unknown_package_ingestion(self):
 
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
-        unknown_pkgs = set(map(lambda pkg: Package(package=pkg.name,
-                                                   version=pkg.version), pkg_list))
+        unknown_pkgs = set(map(lambda pkg: P(package=pkg.name,
+                                             version=pkg.version), pkg_list))
         try:
             unknown_package_flow(ecosystem, unknown_pkgs)
         except Exception as e:

From 1db0e676e4b6dded0b889bfe9d1e356b82726e80 Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Tue, 5 Jan 2021 18:38:10 +0530
Subject: [PATCH 6/8] Added utils as a direct dependency.

---
 requirements.in            | 1 +
 requirements.txt           | 2 +-
 src/v2/stack_aggregator.py | 4 ++--
 tests/requirements.txt     | 4 ++--
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/requirements.in b/requirements.in
index b63c87eb..066fae3f 100644
--- a/requirements.in
+++ b/requirements.in
@@ -10,3 +10,4 @@ psycopg2-binary
 sqlalchemy
 raven[flask]
 f8a_worker @ git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker
+f8a_utils @ git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils
diff --git a/requirements.txt b/requirements.txt
index f8312638..255f36c8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ colorama==0.4.4           # via rainbow-logging-handler
 configobj==5.0.6          # via anymarkup
 cryptography==3.2.1       # via f8a-utils
 dataclasses==0.8          # via pydantic
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via f8a-worker
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via -r requirements.in, f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
 git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker  # via -r requirements.in
 flask-cors==3.0.9         # via -r requirements.in
diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index e9b44158..adc41a53 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -13,8 +13,8 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
-from f8a_utils.ingestion_utils import unknown_package_flow, \
-    Package as P
+from f8a_utils.ingestion_utils import (unknown_package_flow,
+                                       Package as P)
 
 from src.settings import Settings
 from src.utils import (select_latest_version,
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 9d1dd2dd..76386b3a 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile
+#    pip-compile --output-file=requirements.txt requirements.in
 #
 amqp==2.6.1               # via kombu
 anymarkup-core==0.8.1     # via anymarkup
@@ -26,7 +26,7 @@ configobj==5.0.6          # via anymarkup
 coverage==5.3             # via codecov, pytest-cov
 cryptography==3.2.1       # via f8a-utils
 dataclasses==0.8          # via pydantic
-git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via f8a-worker
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@95ebbf2#egg=f8a_utils  # via -r ../requirements.in, f8a-worker
 git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git@8a57ac7#egg=f8a_version_comparator  # via f8a-utils
 git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@cf7b8a7#egg=f8a_worker  # via -r ../requirements.in
 flask-cors==3.0.9         # via -r ../requirements.in

From ebd6e4d00f766743dcf37798176a17f17911b455 Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Thu, 7 Jan 2021 13:36:46 +0530
Subject: [PATCH 7/8] Removed flag for unknown package flow ingestion.

---
 openshift/template.yaml           |  9 ---------
 src/v2/stack_aggregator.py        | 20 ++++++--------------
 tests/v2/test_stack_aggregator.py | 17 -----------------
 3 files changed, 6 insertions(+), 40 deletions(-)

diff --git a/openshift/template.yaml b/openshift/template.yaml
index 418b0052..8fada305 100644
--- a/openshift/template.yaml
+++ b/openshift/template.yaml
@@ -99,8 +99,6 @@ objects:
             value: "metrics-accumulator"
           - name: METRICS_ENDPOINT_URL_PORT
             value: "5200"
-          - name: DISABLE_UNKNOWN_PACKAGE_FLOW
-            value: ${DISABLE_UNKNOWN_PACKAGE_FLOW}
 
           image: "${DOCKER_REGISTRY}/${DOCKER_IMAGE}:${IMAGE_TAG}"
           name: f8a-server-backbone
@@ -226,10 +224,3 @@ parameters:
   required: false
   name: FLASK_LOGGING_LEVEL
   value: "INFO"
-
-- description: Flag to enable or disable the unknown package ingetion flow
-  displayName: DISABLE UNKNOWN PACKAGE FLOW
-  required: false
-  name: DISABLE_UNKNOWN_PACKAGE_FLOW
-  value: "0"
-
diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index adc41a53..86f6155e 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -13,8 +13,7 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
-from f8a_utils.ingestion_utils import (unknown_package_flow,
-                                       Package as P)
+from f8a_utils.ingestion_utils import unknown_package_flow
 
 from src.settings import Settings
 from src.utils import (select_latest_version,
@@ -28,6 +27,7 @@
 from src.v2.normalized_packages import NormalizedPackages, GoNormalizedPackages
 from src.v2.license_service import (get_license_analysis_for_stack,
                                     get_license_service_request_payload)
+from f8a_utils import ingestion_utils
 
 logger = logging.getLogger(__name__)
 _TRUE = ['true', True, 1, '1']
@@ -306,14 +306,10 @@ def get_result(self) -> StackAggregatorResult:
 
     def initiate_unknown_package_ingestion(self):
         """Ingestion of Unknown dependencies."""
-        if Settings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
-            return
-
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
-        unknown_pkgs = set(map(lambda pkg: P(package=pkg.name,
-                                             version=pkg.version), pkg_list))
+        unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
+                                                                   version=pkg.version), pkg_list))
         try:
             unknown_package_flow(ecosystem, unknown_pkgs)
         except Exception as e:
@@ -385,14 +381,10 @@ def __init__(self, request: StackAggregatorRequest = None,
 
     def initiate_unknown_package_ingestion(self):
         """Ingestion of Unknown dependencies."""
-        if Settings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
-            return
-
         ecosystem = self._normalized_packages.ecosystem
         pkg_list = self.get_all_unknown_packages()
-        unknown_pkgs = set(map(lambda pkg: P(package=pkg.name,
-                                             version=pkg.version), pkg_list))
+        unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
+                                                                   version=pkg.version), pkg_list))
         try:
             unknown_package_flow(ecosystem, unknown_pkgs)
         except Exception as e:
diff --git a/tests/v2/test_stack_aggregator.py b/tests/v2/test_stack_aggregator.py
index 841e8260..3b61f21c 100644
--- a/tests/v2/test_stack_aggregator.py
+++ b/tests/v2/test_stack_aggregator.py
@@ -203,23 +203,6 @@ def test_with_2_public_vuln_for_registered(_mock_license, _mock_gremlin):
                vulnerable_dependencies[0].public_vulnerabilities) == 2
 
 
-@mock.patch('src.v2.stack_aggregator.unknown_package_flow')
-@mock.patch('src.v2.stack_aggregator.post_gremlin')
-def test_unknown_flow_with_disabled_flag(_mock_gremlin, _mock_unknown, monkeypatch):
-    """Test unknown flow."""
-    with open("tests/v2/data/graph_response_2_public_vuln.json", "r") as fin:
-        _mock_gremlin.return_value = json.load(fin)
-
-    payload = _request_body()
-    # add unknown package as direct dependency
-    payload['packages'].append(_SIX.dict())
-
-    # Disabled unknown flow check
-    monkeypatch.setenv('DISABLE_UNKNOWN_PACKAGE_FLOW', '1')
-    StackAggregator().execute(payload, persist=False)
-    _mock_unknown.assert_not_called()
-
-
 @mock.patch('src.v2.stack_aggregator.unknown_package_flow')
 @mock.patch('src.v2.stack_aggregator.post_gremlin')
 @mock.patch('src.v2.stack_aggregator.get_license_analysis_for_stack')

From cb2518b7737e4af0691cbc1475b429fc28a481e1 Mon Sep 17 00:00:00 2001
From: jparsai <jparsai@redhat.com>
Date: Fri, 8 Jan 2021 13:53:18 +0530
Subject: [PATCH 8/8] Resolved conflits.

---
 src/v2/stack_aggregator.py | 39 ++++++++++++++++++++++----------------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/src/v2/stack_aggregator.py b/src/v2/stack_aggregator.py
index 9d182cf7..b976e767 100644
--- a/src/v2/stack_aggregator.py
+++ b/src/v2/stack_aggregator.py
@@ -13,9 +13,10 @@
 
 from typing import Dict, List, Tuple, Set
 from f8a_utils.gh_utils import GithubUtils
+from f8a_utils.ingestion_utils import unknown_package_flow
 
 from src.settings import AggregatorSettings
-from src.utils import (select_latest_version, server_create_analysis,
+from src.utils import (select_latest_version,
                        persist_data_in_db, post_gremlin, GREMLIN_QUERY_SIZE,
                        format_date)
 from src.v2.models import (StackAggregatorRequest, GitHubDetails, PackageDetails,
@@ -26,6 +27,7 @@
 from src.v2.normalized_packages import NormalizedPackages, GoNormalizedPackages
 from src.v2.license_service import (get_license_analysis_for_stack,
                                     get_license_service_request_payload)
+from f8a_utils import ingestion_utils
 
 logger = logging.getLogger(__name__)
 _TRUE = ['true', True, 1, '1']
@@ -306,19 +308,17 @@ def get_result(self) -> StackAggregatorResult:
 
     def initiate_unknown_package_ingestion(self):
         """Ingestion of Unknown dependencies."""
-        if AggregatorSettings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
-            return
-
         ecosystem = self._normalized_packages.ecosystem
+        pkg_list = self.get_all_unknown_packages()
+        unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
+                                                                   version=pkg.version), pkg_list))
         try:
-            for dep in self.get_all_unknown_packages():
-                server_create_analysis(ecosystem, dep.name, dep.version, api_flow=True,
-                                       force=False, force_graph_sync=True)
-        except Exception as e:  # pylint:disable=W0703,C0103
-            logger.error('Ingestion failed for {%s, %s, %s}',
-                         ecosystem, dep.name, dep.version)
-            logger.error(e)
+            unknown_package_flow(ecosystem, unknown_pkgs)
+        except Exception as e:
+            logger.error('Unknown ingestion failed with %s', e)
+        else:
+            logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
+                         len(pkg_list), ecosystem)
 
 
 class StackAggregator:
@@ -383,10 +383,17 @@ def __init__(self, request: StackAggregatorRequest = None,
 
     def initiate_unknown_package_ingestion(self):
         """Ingestion of Unknown dependencies."""
-        if AggregatorSettings().disable_unknown_package_flow:
-            logger.warning('Skipping unknown flow %s', self.get_all_unknown_packages())
-            return
-        logger.error('Ingestion is Not active for Golang.')
+        ecosystem = self._normalized_packages.ecosystem
+        pkg_list = self.get_all_unknown_packages()
+        unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
+                                                                   version=pkg.version), pkg_list))
+        try:
+            unknown_package_flow(ecosystem, unknown_pkgs)
+        except Exception as e:
+            logger.error('Unknown ingestion failed with %s', e)
+        else:
+            logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
+                         len(pkg_list), ecosystem)
 
     def _get_package_details_with_vulnerabilities(self) -> List[Dict[str, object]]:
         """Get package data from graph along with vulnerability."""