Update maven pipelines #845

    * Add beta tag to minecode_pipeline version

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

minecode_pipelines/pipelines/__init__.py

@@ -89,7 +89,6 @@ def fetch_federation_config(self):
         data_federation = DataFederation.from_url(
             name="aboutcode-data",
             remote_root_url="https://github.com/aboutcode-data",
-            branch="add-datafile_name",
         )
         self.data_clusters = {
             "purls": data_federation.get_cluster("purls"),

minecode_pipelines/pipelines/mine_maven.py

@@ -89,7 +89,7 @@ def mine_and_publish_maven_packageurls_repo1_maven_org(self):
         _mine_and_publish_packageurls(
             packageurls=self.maven_nexus_collector.get_packages(),
             total_package_count=None,
-            data_cluster=self.data_cluster,
+            data_clusters=self.data_clusters,
             checked_out_repos=self.checked_out_repos,
             working_path=self.working_path,
             append_purls=self.append_purls,
@@ -133,7 +133,7 @@ def mine_and_publish_maven_packageurls_repo_spring_io_release(self):
         _mine_and_publish_packageurls(
             packageurls=self.maven_nexus_collector.get_packages(),
             total_package_count=None,
-            data_cluster=self.data_cluster,
+            data_clusters=self.data_clusters,
             checked_out_repos=self.checked_out_repos,
             working_path=self.working_path,
             append_purls=self.append_purls,
@@ -177,7 +177,7 @@ def mine_and_publish_maven_packageurls_repo_spring_io_milestone(self):
         _mine_and_publish_packageurls(
             packageurls=self.maven_nexus_collector.get_packages(),
             total_package_count=None,
-            data_cluster=self.data_cluster,
+            data_clusters=self.data_clusters,
             checked_out_repos=self.checked_out_repos,
             working_path=self.working_path,
             append_purls=self.append_purls,
@@ -221,7 +221,7 @@ def mine_and_publish_maven_packageurls_plugins_gradle_org(self):
         _mine_and_publish_packageurls(
             packageurls=self.maven_nexus_collector.get_packages(),
             total_package_count=None,
-            data_cluster=self.data_cluster,
+            data_clusters=self.data_clusters,
             checked_out_repos=self.checked_out_repos,
             working_path=self.working_path,
             append_purls=self.append_purls,
@@ -265,7 +265,7 @@ def mine_and_publish_maven_packageurls_repository_apache_org(self):
         _mine_and_publish_packageurls(
             packageurls=self.maven_nexus_collector.get_packages(),
             total_package_count=None,
-            data_cluster=self.data_cluster,
+            data_clusters=self.data_clusters,
             checked_out_repos=self.checked_out_repos,
             working_path=self.working_path,
             append_purls=self.append_purls,

minecode_pipelines/pipes/composer.py

@@ -133,16 +133,6 @@ def mine_composer_packages():
     return load_composer_packages(packages_file)
 
 
-def yield_composer_package_data(packageurls=[]):
-    for purl in packageurls:
-        package_url = PackageURL.from_string(purl)
-        package_data_url = COMPOSER_REPO + "/" + package_url.version + "/" + purl.name + ".json"
-        response = requests.get(package_data_url)
-        if not response.ok:
-            continue
-        yield purl, response.json()
-
-
 def mine_composer_packageurls(packages, start_index):
     """Mine Composer packages from Packagist"""
     packages_iter = cycle_from_index(packages, start_index)
@@ -155,6 +145,5 @@ def mine_composer_packageurls(packages, start_index):
 
             vendor, package = item
             base_purl, packageurls = get_composer_purl(vendor=vendor, package=package)
-            purls_and_package_data = yield_composer_package_data(packageurls)
 
-            yield base_purl, packageurls, purls_and_package_data
+            yield base_purl, packageurls, []

minecode_pipelines/pipes/maven.py

@@ -16,8 +16,10 @@
 
 import arrow
 import javaproperties
+import requests
 from dateutil import tz
 from jawa.util.utf import decode_modified_utf8
+
 from packagedcode.maven import build_filename
 from packagedcode.maven import build_url
 from packagedcode.maven import get_urls
@@ -728,8 +730,17 @@ def _get_packages(self, content=None):
                 namespace=group_id,
                 name=artifact_id,
                 version=version,
+                qualifiers=qualifiers,
+            )
+            packageurls = [package.purl]
+
+            # this yields a tuple containing purl str, dict containing api info
+
+            purls_and_package_data = yield_maven_package_data(
+                purl=current_purl, pom_urls=[api_data_url]
             )
-            yield current_purl, [package.purl]
+
+            yield current_purl, packageurls, purls_and_package_data
 
     def _get_packages_from_index_increments(self):
         for index_increment in self.index_increment_locations:
@@ -743,3 +754,11 @@ def get_packages(self):
         elif self.index_location:
             packages = self._get_packages(content=self.index_location)
         return packages
+
+
+def yield_maven_package_data(purl, pom_urls=[]):
+    for pom_url in pom_urls:
+        response = requests.get(pom_url)
+        if not response.ok:
+            continue
+        yield purl, response.content

pyproject-minecode_pipelines.toml

@@ -1,10 +1,10 @@
 [build-system]
-requires = [ "flot>=0.7.0" ]
+requires = [ "flot>=0.7.3" ]
 build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.1.1"
+version = "1.0.0b1"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }
@@ -38,9 +38,9 @@ classifiers = [
 
 dependencies = [
     "aboutcode.hashid >= 0.2.0",
-    "aboutcode.federated >= 1.0.1",
-    "packageurl_python >= 0.15.6",
-    "scancodeio >= 35.3.0",
+    "aboutcode.federated >= 1.0.3",
+    "packageurl_python >= 0.17.6",
+    "scancodeio >= 37.1.0",
     "ftputil >= 5.1.0",
     "jawa >= 2.2.0",
     "arrow >= 1.3.0",
@@ -64,7 +64,7 @@ mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift"
 mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer"
 
 [tool.bumpversion]
-current_version = "0.1.1"
+current_version = "1.0.0b1"
 allow_dirty = true
 
 files = [