Mine PURL from alpine #665 (#724)

* Add files for alpine pipeline #665

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update AlpineCollector #665

    * Modify functions to return an Alpine PackageData from a parsed pkginfo string

Signed-off-by: Jono Yang <jyang@nexb.com>

* Iteratively download indices and yield packages #665

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update destructor code #665

Signed-off-by: Jono Yang <jyang@nexb.com>

* Create code to collect and write purl files #665

Signed-off-by: Jono Yang <jyang@nexb.com>

* Add append argument to write_packageurls_to_file #665

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update code formatting #665

    * move single function to alpine pipe

Signed-off-by: Jono Yang <jyang@nexb.com>

* Add test for alpine pipe #665

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update logic in MavenCollector constructor #660

Signed-off-by: Jono Yang <jyang@nexb.com>

* Update test expectation

Signed-off-by: Jono Yang <jyang@nexb.com>

---------

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

minecode/tests/collectors/test_github.py

@@ -35,6 +35,11 @@ def test_github_get_all_versions(self):
             "matchcode-toolkit-v3.0.0",
             "matchcode-toolkit-v1.1.1",
             "minecode-pipelines/v0.0.1b3",
+            "minecode-pipelines/v0.0.1b4",
+            "minecode-pipelines/v0.0.1b5",
+            "minecode-pipelines/v0.0.1b6",
+            "minecode-pipelines/v0.0.1b7",
+            "minecode-pipelines/v0.0.1b8",
         ]
         for item in versions:
             self.assertIn(item, expected)

minecode_pipelines/pipelines/mine_alpine.py

@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import federatedcode
+
+from minecode_pipelines import pipes
+from minecode_pipelines.pipes import alpine
+
+
+class MineAlpine(Pipeline):
+    """
+    Mine all packageURLs from an alpine index and publish them to
+    a FederatedCode repo.
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.collect_packages_from_alpine,
+            cls.delete_cloned_repos,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
+
+    def collect_packages_from_alpine(self):
+        self.repos = alpine.collect_packages_from_alpine(logger=self.log)
+
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(repos=self.repos, logger=self.log)

minecode_pipelines/pipelines/mine_pypi.py

@@ -23,8 +23,8 @@
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import federatedcode
 
-from minecode_pipelines.pipes import pypi
 from minecode_pipelines import pipes
+from minecode_pipelines.pipes import pypi
 
 
 class MinePypi(Pipeline):

minecode_pipelines/pipes/__init__.py

@@ -81,13 +81,27 @@ def update_mined_packages_in_checkpoint(packages, config_repo, cloned_repo, chec
     )
 
 
-def write_packageurls_to_file(repo, base_dir, packageurls):
+def write_packageurls_to_file(repo, base_dir, packageurls, append=False):
+    if not isinstance(packageurls, list):
+        raise Exception("`packageurls` needs to be a list")
+
     purl_file_rel_path = os.path.join(base_dir, PURLS_FILENAME)
     purl_file_full_path = Path(repo.working_dir) / purl_file_rel_path
+    if append and purl_file_full_path.exists():
+        existing_purls = load_data_from_yaml_file(purl_file_full_path)
+        packageurls = existing_purls.extend(packageurls)
     write_data_to_yaml_file(path=purl_file_full_path, data=packageurls)
     return purl_file_rel_path
 
 
+def load_data_from_yaml_file(path):
+    if isinstance(path, str):
+        path = Path(path)
+
+    with open(path, encoding="utf-8") as f:
+        return saneyaml.load(f.read())
+
+
 def write_data_to_yaml_file(path, data):
     if isinstance(path, str):
         path = Path(path)