Fix cpan issues and release cpan pipeline

Signed-off-by: Ayan Sinha Mahapatra <asmahapatra@aboutcode.org>

Author: AyanSinhaMahapatra

minecode_pipelines/__init__.py

@@ -8,4 +8,4 @@
 #
 
 
-VERSION = "0.0.1b42"
+VERSION = "0.0.1b59"

minecode_pipelines/miners/cpan.py

@@ -13,8 +13,7 @@
 from bs4 import BeautifulSoup
 from packageurl import PackageURL
 
-from scanpipe.pipes.fetch import fetch_http
-
+from minecode_pipelines.utils import get_temp_file
 
 """
 Visitors for cpan and cpan-like perl package repositories.
@@ -33,9 +32,19 @@ def get_cpan_packages(cpan_repo=CPAN_REPO, logger=None):
     path_prefixes with author page path from this list.
     """
     cpan_packages_url = cpan_repo + "modules/02packages.details.txt.gz"
-    cpan_packages_gz_download = fetch_http(cpan_packages_url)
-    with gzip.open(cpan_packages_gz_download, "rb") as file_content:
-        packages_content = file_content.read()
+    packages_archive = get_temp_file(file_name="cpan_packages", extension=".gz")
+    packages_content = get_temp_file(file_name="cpan_packages", extension=".txt")
+    response = requests.get(cpan_packages_url, stream=True)
+    with open(packages_archive, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            f.write(chunk)
+
+    with gzip.open(packages_archive, "rb") as f_in:
+        with open(packages_content, "wb") as f_out:
+            f_out.writelines(f_in)
+
+    with open(packages_content, 'r', encoding='utf-8') as file:
+        packages_content = file.read()
 
     package_path_by_name = {}
 

minecode_pipelines/pipelines/mine_cpan.py

@@ -40,7 +40,6 @@ def steps(cls):
             cls.create_federatedcode_working_dir,
             cls.mine_cpan_packages,
             cls.fetch_federation_config,
-            cls.mine_and_publish_cpan_packageurls,
             cls.mine_and_publish_packageurls,
             cls.delete_working_dir,
         )
@@ -54,7 +53,8 @@ def packages_count(self):
 
     def mine_packageurls(self):
         """Get cpan packageURLs for all mined cpan package names."""
-        cpan.mine_and_publish_cpan_packageurls(
+        yield from cpan.mine_and_publish_cpan_packageurls(
             package_path_by_name=self.cpan_packages_path_by_name,
             logger=self.log,
         )
+

minecode_pipelines/pipes/cpan.py

@@ -27,7 +27,6 @@
 from minecode_pipelines.miners.cpan import CPAN_TYPE
 from minecode_pipelines.utils import grouper
 
-from aboutcode.hashid import get_package_base_dir
 from packageurl import PackageURL
 
 # If True, show full details on fetching packageURL for
@@ -54,7 +53,6 @@ def mine_and_publish_cpan_packageurls(package_path_by_name, logger=None):
     if not package_path_by_name:
         return
 
-    packageurls_by_base_purl = {}
     for package_batch in grouper(n=PACKAGE_BATCH_SIZE, iterable=package_path_by_name.keys()):
         packages_mined = []
 
@@ -94,6 +92,4 @@ def mine_and_publish_cpan_packageurls(package_path_by_name, logger=None):
                 logger(f"packageURLs: {purls_string}")
 
             packages_mined.append(package_name)
-            packageurls_by_base_purl[base_purl] = packageurls
-    
-    return packageurls_by_base_purl
+            yield base_purl, packageurls

pyproject-minecode_pipelines.toml

@@ -4,7 +4,7 @@ build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.0.1b42"
+version = "0.0.1b59"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }
@@ -63,7 +63,7 @@ mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift"
 mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer"
 
 [tool.bumpversion]
-current_version = "0.0.1b42"
+current_version = "0.0.1b59"
 allow_dirty = true
 
 files = [