Use configs for source symbol extensions

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

scanpipe/pipelines/deploy_to_develop.py

@@ -156,7 +156,7 @@ def match_archives_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_package_extensions,
+            extensions=self.matchable_package_extensions,
             matcher_func=d2d.match_purldb_package,
             logger=self.log,
         )
@@ -234,7 +234,7 @@ def match_resources_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_resource_extensions,
+            extensions=self.matchable_resource_extensions,
             matcher_func=d2d.match_purldb_resource,
             logger=self.log,
         )
@@ -290,7 +290,7 @@ def perform_house_keeping_tasks(self):
         )
         d2d.match_unmapped_resources(
             project=self.project,
-            matched_extensions=self.ecosystem_config.purldb_resource_extensions,
+            matched_extensions=self.ecosystem_config.matchable_resource_extensions,
             logger=self.log,
         )
         d2d.flag_undeployed_resources(project=self.project)

scanpipe/pipes/d2d.py

@@ -54,6 +54,7 @@
 from scanpipe.models import CodebaseRelation
 from scanpipe.models import CodebaseResource
 from scanpipe.models import convert_glob_to_django_regex
+from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import get_resource_diff_ratio
 from scanpipe.pipes import js
@@ -1939,7 +1940,10 @@ def map_rust_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from rust source files
-    rust_from_resources = from_resources.filter(extension=".rs")
+    rust_config = d2d_config.get_ecosystem_config(ecosystem="Rust")
+    rust_from_resources = from_resources.filter(
+        extension__in=rust_config.source_symbol_extensions
+    )
 
     map_binaries_with_symbols(
         project=project,
@@ -1959,7 +1963,10 @@ def map_elfs_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from elf related source files
-    elf_from_resources = from_resources.filter(extension__in=[".c", ".cpp", ".h"])
+    elf_config = d2d_config.get_ecosystem_config(ecosystem="Elf")
+    elf_from_resources = from_resources.filter(
+        extension__in=elf_config.source_symbol_extensions
+    )
 
     map_binaries_with_symbols(
         project=project,
@@ -1982,8 +1989,9 @@ def map_macho_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from macos related source files
+    macos_config = d2d_config.get_ecosystem_config(ecosystem="MacOS")
     mac_from_resources = from_resources.filter(
-        extension__in=[".c", ".cpp", ".h", ".m", ".swift"]
+        extension__in=macos_config.source_symbol_extensions,
     )
 
     map_binaries_with_symbols(
@@ -2004,8 +2012,9 @@ def map_winpe_binaries_with_symbols(project, logger=None):
     )
 
     # Collect source symbols from windows related source files
+    windows_config = d2d_config.get_ecosystem_config(ecosystem="Windows")
     windows_from_resources = from_resources.filter(
-        extension__in=[".c", ".cpp", ".h", ".cs"]
+        extension__in=windows_config.source_symbol_extensions,
     )
 
     map_binaries_with_symbols(
@@ -2071,16 +2080,17 @@ def map_javascript_symbols(project, logger=None):
     """Map deployed JavaScript, TypeScript to its sources using symbols."""
     project_files = project.codebaseresources.files()
 
+    js_config = d2d_config.get_ecosystem_config(ecosystem="JavaScript")
     javascript_to_resources = (
         project_files.to_codebase()
         .has_no_relation()
-        .filter(extension__in=[".ts", ".js"])
+        .filter(extension__in=js_config.source_symbol_extensions)
     )
 
     javascript_from_resources = (
         project_files.from_codebase()
         .exclude(path__contains="/test/")
-        .filter(extension__in=[".ts", ".js"])
+        .filter(extension__in=js_config.source_symbol_extensions)
     )
 
     if not (javascript_from_resources.exists() and javascript_to_resources.exists()):

scanpipe/pipes/d2d_config.py

@@ -36,13 +36,13 @@ class EcosystemConfig:
     # are options in the pipelines
     ecosystem_option: str = "Default"
 
-    # These are extensions for packages of this ecosystem which
-    # need to be matched from purldb
-    purldb_package_extensions: list = field(default_factory=list)
+    # These are extensions for package archive files for this ecosystem
+    # that are matchable against the purldb using matchcode
+    matchable_package_extensions: list = field(default_factory=list)
 
-    # These are extensions for resources of this ecosystem which
-    # need to be matched from purldb
-    purldb_resource_extensions: list = field(default_factory=list)
+    # These are extensions for file of this ecosystem that are
+    # matchable against the purldb using matchcode
+    matchable_resource_extensions: list = field(default_factory=list)
 
     # Extensions for document files which do not require review
     doc_extensions: list = field(default_factory=list)
@@ -60,11 +60,15 @@ class EcosystemConfig:
     # which are not so useful in mapping
     standard_symbols_to_exclude: list = field(default_factory=list)
 
+    # File extesions which should be looked at for source symbol extraction
+    # for mapping using symbols for a specific selected option/ecosystem
+    source_symbol_extensions: list = field(default_factory=list)
+
 
 # Dictionary of ecosystem configurations
 ECOSYSTEM_CONFIGS = {
     "Default": EcosystemConfig(
-        purldb_package_extensions=[".zip", ".tar.gz", ".tar.xz"],
+        matchable_package_extensions=[".zip", ".tar.gz", ".tar.xz"],
         devel_resource_path_exclusions=["*/tests/*"],
         doc_extensions=[
             ".pdf",
@@ -79,12 +83,12 @@ class EcosystemConfig:
     ),
     "Java": EcosystemConfig(
         ecosystem_option="Java",
-        purldb_package_extensions=[".jar", ".war"],
-        purldb_resource_extensions=[".class"],
+        matchable_package_extensions=[".jar", ".war"],
+        matchable_resource_extensions=[".class"],
     ),
     "JavaScript": EcosystemConfig(
         ecosystem_option="JavaScript",
-        purldb_resource_extensions=[
+        matchable_resource_extensions=[
             ".map",
             ".js",
             ".mjs",
@@ -98,21 +102,35 @@ class EcosystemConfig:
             ".sass",
             ".soy",
         ],
+        source_symbol_extensions=[".ts", ".js"],
     ),
     "Go": EcosystemConfig(
         ecosystem_option="Go",
-        purldb_resource_extensions=[".go"],
+        matchable_resource_extensions=[".go"],
     ),
     "Rust": EcosystemConfig(
         ecosystem_option="Rust",
-        purldb_resource_extensions=[".rs"],
+        matchable_resource_extensions=[".rs"],
+        source_symbol_extensions=[".rs"],
     ),
     "Ruby": EcosystemConfig(
         ecosystem_option="Ruby",
-        purldb_package_extensions=[".gem"],
-        purldb_resource_extensions=[".rb"],
+        matchable_package_extensions=[".gem"],
+        matchable_resource_extensions=[".rb"],
         deployed_resource_path_exclusions=["*checksums.yaml.gz*", "*metadata.gz*"],
     ),
+    "Elf": EcosystemConfig(
+        ecosystem_option="Elf",
+        source_symbol_extensions=[".c", ".cpp", ".h"],
+    ),
+    "MacOS": EcosystemConfig(
+        ecosystem_option="MacOS",
+        source_symbol_extensions=[".c", ".cpp", ".h", ".m", ".swift"],
+    ),
+    "Windows": EcosystemConfig(
+        ecosystem_option="Windows",
+        source_symbol_extensions=[".c", ".cpp", ".h", ".cs"],
+    ),
 }
 
 
@@ -150,8 +168,8 @@ def add_ecosystem_config(pipeline_ecosystem_config, ecosystem_config):
     based configuration defined in `ecosystem_config`.
     """
     d2d_pipeline_configs = [
-        "purldb_package_extensions",
-        "purldb_resource_extensions",
+        "matchable_package_extensions",
+        "matchable_resource_extensions",
         "deployed_resource_path_exclusions",
     ]