Update npm packageURL mining pipeline

* Fix npm packages JSON file structure
* Compress package names JSON to reduce size and allow git push

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

minecode_pipelines/miners/npm.py

@@ -106,7 +106,7 @@ def get_npm_packages(replicate_url=NPM_REPLICATE_REPO):
 
     package_data = response.json()
     package_names, last_key = get_package_names_last_key(package_data)
-    all_package_names.append(package_names)
+    all_package_names.extend(package_names)
 
     total_rows = package_data.get("total_rows")
     iterations = int(total_rows / NPM_REPLICATE_BATCH_SIZE) + 1
@@ -121,7 +121,7 @@ def get_npm_packages(replicate_url=NPM_REPLICATE_REPO):
 
         package_data = response.json()
         package_names, last_key = get_package_names_last_key(package_data)
-        all_package_names.append(package_names)
+        all_package_names.extend(package_names)
 
     return {"packages": all_package_names}
 

minecode_pipelines/pipes/__init__.py

@@ -7,8 +7,10 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import gzip
 import json
 import os
+import shutil
 from pathlib import Path
 
 import requests
@@ -31,6 +33,18 @@
 MINECODE_PIPELINES_CONFIG_REPO = "https://github.com/aboutcode-data/minecode-pipelines-config/"
 
 
+def compress_packages_file(packages_file, compressed_packages_file):
+    with open(packages_file, "rb") as f_in:
+        with gzip.open(compressed_packages_file, "wb") as f_out:
+            f_out.writelines(f_in)
+
+
+def decompress_packages_file(packages_file, compressed_packages_file):
+    with gzip.open(compressed_packages_file, "rb") as f_in:
+        with open(packages_file, "wb") as f_out:
+            f_out.writelines(f_in)
+
+
 def write_packages_json(packages, name):
     temp_file = get_temp_file(name)
     write_data_to_json_file(path=temp_file, data=packages)
@@ -68,6 +82,17 @@ def update_checkpoints_in_github(checkpoint, cloned_repo, path):
     )
 
 
+def update_checkpoints_file_in_github(checkpoints_file, cloned_repo, path):
+    checkpoint_path = os.path.join(cloned_repo.working_dir, path)
+    shutil.move(checkpoints_file, checkpoint_path)
+    commit_message = """Update federatedcode purl mining checkpoint"""
+    commit_and_push_changes(
+        repo=cloned_repo,
+        files_to_commit=[checkpoint_path],
+        commit_message=commit_message,
+    )
+
+
 def get_mined_packages_from_checkpoint(config_repo, checkpoint_path):
     checkpoint = fetch_checkpoint_from_github(
         config_repo=config_repo,

minecode_pipelines/pipes/npm.py

@@ -26,6 +26,7 @@
 from minecode_pipelines.pipes import write_packageurls_to_file
 from minecode_pipelines.pipes import fetch_checkpoint_from_github
 from minecode_pipelines.pipes import update_checkpoints_in_github
+from minecode_pipelines.pipes import update_checkpoints_file_in_github
 from minecode_pipelines.pipes import get_mined_packages_from_checkpoint
 from minecode_pipelines.pipes import update_mined_packages_in_checkpoint
 from minecode_pipelines.pipes import get_packages_file_from_checkpoint
@@ -34,6 +35,8 @@
 from minecode_pipelines.pipes import INITIAL_SYNC_STATE
 from minecode_pipelines.pipes import PERIODIC_SYNC_STATE
 from minecode_pipelines.pipes import write_packages_json
+from minecode_pipelines.pipes import compress_packages_file
+from minecode_pipelines.pipes import decompress_packages_file
 
 
 from minecode_pipelines.miners.npm import get_npm_packages
@@ -57,7 +60,9 @@
 
 
 PACKAGE_FILE_NAME = "NPMPackages.json"
+COMPRESSED_PACKAGE_FILE_NAME = "NPMPackages.json.gz"
 NPM_REPLICATE_CHECKPOINT_PATH = "npm/" + PACKAGE_FILE_NAME
+COMPRESSED_NPM_REPLICATE_CHECKPOINT_PATH = "npm/" + COMPRESSED_PACKAGE_FILE_NAME
 NPM_CHECKPOINT_PATH = "npm/checkpoints.json"
 NPM_PACKAGES_CHECKPOINT_PATH = "npm/packages_checkpoint.json"
 
@@ -103,10 +108,15 @@ def mine_npm_packages(logger=None):
             packages=packages,
             name=PACKAGE_FILE_NAME,
         )
-        update_checkpoints_in_github(
-            checkpoint=packages,
+        compressed_packages_file = packages_file + ".gz"
+        compress_packages_file(
+            packages_file=packages_file,
+            compressed_packages_file=compressed_packages_file,
+        )
+        update_checkpoints_file_in_github(
+            checkpoints_file=compressed_packages_file,
             cloned_repo=cloned_repo,
-            path=NPM_REPLICATE_CHECKPOINT_PATH,
+            path=COMPRESSED_NPM_REPLICATE_CHECKPOINT_PATH,
         )
 
         if logger:
@@ -129,10 +139,15 @@ def mine_npm_packages(logger=None):
             settings_path=NPM_CHECKPOINT_PATH,
         )
 
-        packages_file = get_packages_file_from_checkpoint(
+        compressed_packages_file = get_packages_file_from_checkpoint(
             config_repo=MINECODE_PIPELINES_CONFIG_REPO,
-            checkpoint_path=NPM_REPLICATE_CHECKPOINT_PATH,
-            name=PACKAGE_FILE_NAME,
+            checkpoint_path=COMPRESSED_NPM_REPLICATE_CHECKPOINT_PATH,
+            name=COMPRESSED_PACKAGE_FILE_NAME,
+        )
+        packages_file = compressed_packages_file.replace(".gz", "")
+        decompress_packages_file(
+            packages_file=packages_file,
+            compressed_packages_file=compressed_packages_file,
         )
 
     elif state == PERIODIC_SYNC_STATE: