fix: exclude certain file patterns from map_deploy_to_develop (Java) (#1995)

Signed-off-by: Chin Yeung Li <tli@nexb.com>

Author: chinyeungli

scanpipe/pipes/d2d_config.py

@@ -80,22 +80,37 @@ class EcosystemConfig:
             ".odt",
             ".odp",
         ],
+        deployed_resource_path_exclusions=["*.properties", "*.html"],
     ),
     "Java": EcosystemConfig(
         ecosystem_option="Java",
         matchable_package_extensions=[".jar", ".war"],
         matchable_resource_extensions=[".class"],
-        deployed_resource_path_exclusions=["*META-INF/*", "*/module-info.class"],
+        deployed_resource_path_exclusions=[
+            "*META-INF/*",
+            "*/module-info.class",
+            "*/OSGI-INF/*.xml",
+            "*/OSGI-INF/*.json",
+            "*spring-configuration-metadata.json",
+        ],
     ),
     "Scala": EcosystemConfig(
         ecosystem_option="Scala",
         matchable_package_extensions=[".jar", ".war"],
         matchable_resource_extensions=[".class"],
+        deployed_resource_path_exclusions=[
+            "*META-INF/*",
+        ],
     ),
     "Kotlin": EcosystemConfig(
         ecosystem_option="Kotlin",
         matchable_package_extensions=[".jar", ".war"],
         matchable_resource_extensions=[".class"],
+        deployed_resource_path_exclusions=[
+            "*META-INF/*",
+            "*.knm",
+            "*kotlin-project-structure-metadata.json",
+        ],
     ),
     "JavaScript": EcosystemConfig(
         ecosystem_option="JavaScript",

scanpipe/tests/data/d2d/config/ecosystem_config.json

@@ -3,7 +3,7 @@
     "matchable_package_extensions": [".jar", ".war", ".gem", ".zip", ".tar.gz", ".tar.xz"],
     "matchable_resource_extensions": [".map", ".js", ".mjs", ".ts", ".d.ts", ".jsx", ".tsx", ".css", ".scss", ".less", ".sass", ".soy",".class", ".rb"],
     "doc_extensions": [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".tex", ".odt", ".odp"],
-    "deployed_resource_path_exclusions": ["*META-INF/*", "*/module-info.class", "*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig"],
+    "deployed_resource_path_exclusions": ["*META-INF/*", "*/module-info.class", "*/OSGI-INF/*.xml", "*/OSGI-INF/*.json", "*spring-configuration-metadata.json", "*checksums.yaml.gz*", "*metadata.gz*", "*data.tar.gz.sig", "*.properties", "*.html"],
     "devel_resource_path_exclusions": ["*/tests/*"],
     "standard_symbols_to_exclude": [],
     "source_symbol_extensions": []

scanpipe/tests/pipes/test_d2d.py

@@ -503,6 +503,12 @@ def test_scanpipe_pipes_d2d_java_ignore_pattern(self):
         make_resource_file(self.project1, path="to/META-INF/MANIFEST.MF")
         make_resource_file(self.project1, path="to/test.class")
         make_resource_file(self.project1, path="to/META-INF/others.txt")
+        make_resource_file(
+            self.project1, path="to/META-INF/spring-configuration-metadata.json"
+        )
+        make_resource_file(self.project1, path="to/OSGI-INF/test.xml")
+        make_resource_file(self.project1, path="to/OSGI-INF/test.json")
+        make_resource_file(self.project1, path="to/OSGI-INF/test.class")
         buffer = io.StringIO()
 
         java_config = d2d_config.get_ecosystem_config(ecosystem="Java")
@@ -511,7 +517,7 @@ def test_scanpipe_pipes_d2d_java_ignore_pattern(self):
             patterns_to_ignore=java_config.deployed_resource_path_exclusions,
             logger=buffer.write,
         )
-        expected = "Ignoring 3 to/ resources with ecosystem specific configurations."
+        expected = "Ignoring 6 to/ resources with ecosystem specific configurations."
         self.assertIn(expected, buffer.getvalue())
 
     def test_scanpipe_pipes_d2d_map_jar_to_java_source(self):
@@ -612,6 +618,21 @@ def test_scanpipe_pipes_d2d_map_jar_to_scala_source(self):
         self.assertEqual(from2, relation.from_resource)
         self.assertEqual(to_jar, relation.to_resource)
 
+    def test_scanpipe_pipes_d2d_scala_ignore_pattern(self):
+        make_resource_file(self.project1, path="to/META-INF/MANIFEST.MF")
+        make_resource_file(self.project1, path="to/test.class")
+        make_resource_file(self.project1, path="to/META-INF/others.txt")
+        buffer = io.StringIO()
+
+        scala_config = d2d_config.get_ecosystem_config(ecosystem="Scala")
+        d2d.ignore_unmapped_resources_from_config(
+            project=self.project1,
+            patterns_to_ignore=scala_config.deployed_resource_path_exclusions,
+            logger=buffer.write,
+        )
+        expected = "Ignoring 2 to/ resources with ecosystem specific configurations."
+        self.assertIn(expected, buffer.getvalue())
+
     def test_scanpipe_pipes_d2d_map_jar_to_kotlin_source(self):
         from1 = make_resource_file(
             self.project1,
@@ -661,6 +682,23 @@ def test_scanpipe_pipes_d2d_map_jar_to_kotlin_source(self):
         self.assertEqual(from2, relation.from_resource)
         self.assertEqual(to_jar, relation.to_resource)
 
+    def test_scanpipe_pipes_d2d_kotlin_ignore_pattern(self):
+        make_resource_file(self.project1, path="to/META-INF/test.knm")
+        make_resource_file(self.project1, path="to/test.class")
+        make_resource_file(
+            self.project1, path="to/META-INF/kotlin-project-structure-metadata.json"
+        )
+        buffer = io.StringIO()
+
+        kotlin_config = d2d_config.get_ecosystem_config(ecosystem="Kotlin")
+        d2d.ignore_unmapped_resources_from_config(
+            project=self.project1,
+            patterns_to_ignore=kotlin_config.deployed_resource_path_exclusions,
+            logger=buffer.write,
+        )
+        expected = "Ignoring 2 to/ resources with ecosystem specific configurations."
+        self.assertIn(expected, buffer.getvalue())
+
     def test_scanpipe_pipes_d2d_map_jar_to_source_works_for_jar(self):
         from1 = make_resource_file(
             self.project1,