Merge resolved

Author: rejzzzz

README.rst

@@ -57,9 +57,8 @@ for any legal advice.
 
 
 
-
-.. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml/badge.svg?branch=main
-    :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml
+.. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/run-unit-tests.yml/badge.svg?branch=main
+    :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/run-unit-tests.yml
     :alt: CI Tests Status
 
 .. |docs-rtd| image:: https://readthedocs.org/projects/scancodeio/badge/?version=latest

pyproject.toml

@@ -123,7 +123,7 @@ android_analysis = [
   "android_inspector==0.0.1"
 ]
 mining = [
-  "minecode_pipelines==0.0.1b8"
+  "minecode_pipelines==0.1.1"
 ]
 
 [project.urls]

scancodeio/static/main.css

@@ -391,6 +391,12 @@ progress.file-upload::before {
 #message-list th#column-severity {
   min-width: 110px;
 }
+th#column-vulnerability_id {
+  min-width: 220px;
+}
+th#column-summary {
+  width: 40%;
+}
 .menu.is-info .is-active {
   background-color: #3e8ed0;
 }

scanpipe/management/commands/check-compliance.py

@@ -104,23 +104,17 @@ def check_compliance(self, fail_level):
         return total_issues > 0
 
     def check_vulnerabilities(self):
-        packages = self.project.discoveredpackages.vulnerable_ordered()
-        dependencies = self.project.discovereddependencies.vulnerable_ordered()
-
-        vulnerable_records = list(packages) + list(dependencies)
-        count = len(vulnerable_records)
+        all_vulnerabilities = self.project.vulnerabilities
+        vulnerabilities_count = len(all_vulnerabilities)
 
         if self.verbosity > 0:
-            if count:
-                self.stderr.write(f"{count} vulnerable records found:")
-                for entry in vulnerable_records:
-                    self.stderr.write(str(entry))
-                    vulnerability_ids = [
-                        vulnerability.get("vulnerability_id")
-                        for vulnerability in entry.affected_by_vulnerabilities
-                    ]
-                    self.stderr.write(" > " + ", ".join(vulnerability_ids))
+            if vulnerabilities_count:
+                self.stderr.write(f"{vulnerabilities_count} vulnerabilities found:")
+                for vulnerability_id, vulnerability_data in all_vulnerabilities.items():
+                    self.stderr.write(str(vulnerability_id))
+                    for affected_obj in vulnerability_data.get("affects", []):
+                        self.stderr.write(f" > {affected_obj}")
             else:
                 self.stdout.write("No vulnerabilities found")
 
-        return count > 0
+        return vulnerabilities_count > 0

scanpipe/models.py

@@ -1495,6 +1495,11 @@ def vulnerable_dependency_count(self):
         """Return the number of vulnerable dependencies related to this project."""
         return self.vulnerable_dependencies.count()
 
+    @cached_property
+    def vulnerability_count(self):
+        """Return the number of vulnerabilities related to this project."""
+        return self.vulnerable_package_count + self.vulnerable_dependency_count
+
     @cached_property
     def dependency_count(self):
         """Return the number of dependencies related to this project."""

scanpipe/pipelines/deploy_to_develop.py

@@ -83,6 +83,15 @@ def steps(cls):
             cls.find_grammar_packages,
             cls.map_grammar_to_class,
             cls.map_jar_to_grammar_source,
+            cls.find_groovy_packages,
+            cls.map_groovy_to_class,
+            cls.map_jar_to_groovy_source,
+            cls.find_aspectj_packages,
+            cls.map_aspectj_to_class,
+            cls.map_jar_to_aspectj_source,
+            cls.find_clojure_packages,
+            cls.map_clojure_to_class,
+            cls.map_jar_to_clojure_source,
             cls.find_xtend_packages,
             cls.map_xtend_to_class,
             cls.map_javascript,
@@ -230,7 +239,7 @@ def find_scala_packages(self):
 
     @optional_step("Scala")
     def map_scala_to_class(self):
-        """Map a .class compiled file to its .java source."""
+        """Map a .class compiled file to its .scala source."""
         self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.ScalaLanguage)
 
     @optional_step("Scala")
@@ -240,14 +249,14 @@ def map_jar_to_scala_source(self):
 
     @optional_step("Kotlin")
     def find_kotlin_packages(self):
-        """Find the java package of the .java source files."""
+        """Find the java package of the kotlin source files."""
         d2d.find_jvm_packages(
             project=self.project, jvm_lang=jvm.KotlinLanguage, logger=self.log
         )
 
     @optional_step("Kotlin")
     def map_kotlin_to_class(self):
-        """Map a .class compiled file to its .java source."""
+        """Map a .class compiled file to its kotlin source."""
         self.run_d2d_step(d2d.map_jvm_to_class, jvm_lang=jvm.KotlinLanguage)
 
     @optional_step("Kotlin")
@@ -272,6 +281,69 @@ def map_jar_to_grammar_source(self):
         """Map .jar files to their related source directory."""
         self.run_d2d_step(d2d.map_jar_to_jvm_source, jvm_lang=jvm.GrammarLanguage)
 
+    @optional_step("Groovy")
+    def find_groovy_packages(self):
+        """Find the package of the .groovy source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log
+        )
+
+    @optional_step("Groovy")
+    def map_groovy_to_class(self):
+        """Map a .class compiled file to its .groovy source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log
+        )
+
+    @optional_step("Groovy")
+    def map_jar_to_groovy_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.GroovyLanguage, logger=self.log
+        )
+
+    @optional_step("AspectJ")
+    def find_aspectj_packages(self):
+        """Find the package of the .aj source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log
+        )
+
+    @optional_step("AspectJ")
+    def map_aspectj_to_class(self):
+        """Map a .class compiled file to its .aj source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log
+        )
+
+    @optional_step("AspectJ")
+    def map_jar_to_aspectj_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.AspectJLanguage, logger=self.log
+        )
+
+    @optional_step("Clojure")
+    def find_clojure_packages(self):
+        """Find the package of the .clj source files."""
+        d2d.find_jvm_packages(
+            project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log
+        )
+
+    @optional_step("Clojure")
+    def map_clojure_to_class(self):
+        """Map a .class compiled file to its .clj source."""
+        d2d.map_jvm_to_class(
+            project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log
+        )
+
+    @optional_step("Clojure")
+    def map_jar_to_clojure_source(self):
+        """Map .jar files to their related source directory."""
+        d2d.map_jar_to_jvm_source(
+            project=self.project, jvm_lang=jvm.ClojureLanguage, logger=self.log
+        )
+
     @optional_step("Xtend")
     def find_xtend_packages(self):
         """Find the java package of the xtend source files."""

scanpipe/pipes/cyclonedx.py

@@ -30,6 +30,7 @@
 from cyclonedx.model import license as cdx_license_model
 from cyclonedx.model.bom import Bom
 from cyclonedx.schema import SchemaVersion
+from cyclonedx.schema.schema import BaseSchemaVersion
 from cyclonedx.validation import ValidationError
 from cyclonedx.validation.json import JsonStrictValidator
 from defusedxml import ElementTree as SafeElementTree
@@ -184,10 +185,12 @@ def cyclonedx_component_to_package_data(
     affected_by_vulnerabilities = []
     if affected_by := vulnerabilities.get(bom_ref):
         for cdx_vulnerability in affected_by:
+            cdx_vulnerability_json = cdx_vulnerability.as_json(view_=BaseSchemaVersion)
             affected_by_vulnerabilities.append(
                 {
                     "vulnerability_id": str(cdx_vulnerability.id),
                     "summary": cdx_vulnerability.description,
+                    "cdx_vulnerability_data": json.loads(cdx_vulnerability_json),
                 }
             )
 

scanpipe/pipes/d2d.py

@@ -174,12 +174,21 @@ def _map_jvm_to_class_resource(
     from/ fully qualified binary files.
     """
     for extension in jvm_lang.source_extensions:
-        normalized_path = jvm_lang.get_normalized_path(
+        # Perform basic conversion from .class to source file path
+        source_path = jvm_lang.get_source_path(
             path=to_resource.path, extension=extension
         )
-        match = pathmap.find_paths(path=normalized_path, index=from_classes_index)
+        # Perform basic mapping without normalization for scenarios listed in
+        # https://github.com/aboutcode-org/scancode.io/issues/1873
+        match = pathmap.find_paths(path=source_path, index=from_classes_index)
+
         if not match:
-            return
+            normalized_path = jvm_lang.get_normalized_path(
+                path=to_resource.path, extension=extension
+            )
+            match = pathmap.find_paths(path=normalized_path, index=from_classes_index)
+            if not match:
+                return
 
         for resource_id in match.resource_ids:
             from_resource = from_resources.get(id=resource_id)
@@ -240,8 +249,8 @@ def map_jvm_to_class(
 
     if logger:
         logger(
-            f"Mapping {to_resource_count:,d} .class resources to "
-            f"{from_resource_count:,d} {jvm_lang.source_extensions}"
+            f"Mapping {to_resource_count:,d} .class (or other deployed file) "
+            f"resources to {from_resource_count:,d} {jvm_lang.source_extensions}"
         )
 
     # build an index using from-side fully qualified class file names

scanpipe/pipes/d2d_config.py

@@ -112,6 +112,14 @@ class EcosystemConfig:
             "*kotlin-project-structure-metadata.json",
         ],
     ),
+    "Groovy": EcosystemConfig(
+        ecosystem_option="Groovy",
+        matchable_package_extensions=[".jar", ".war"],
+        matchable_resource_extensions=[".class"],
+        deployed_resource_path_exclusions=[
+            "*META-INF/*",
+        ],
+    ),
     "JavaScript": EcosystemConfig(
         ecosystem_option="JavaScript",
         matchable_resource_extensions=[

scanpipe/pipes/jvm.py

@@ -140,12 +140,27 @@ def get_normalized_path(cls, path, extension):
         # https://github.com/aboutcode-org/scancode.io/issues/1994
         if class_name.endswith("_$logger.class"):
             class_name, _, _ = class_name.partition("_$logger.class")
-        elif "$" in class_name:  # inner class
+        elif "$" in class_name and not class_name.startswith("$"):  # inner class
             class_name, _, _ = class_name.partition("$")
         else:
             class_name, _, _ = class_name.partition(".")  # plain .class
         return str(path.parent / f"{class_name}{extension}")
 
+    @classmethod
+    def get_source_path(cls, path, extension):
+        """
+        Return a JVM file path for ``path`` .class file path string.
+        No normalization is performed.
+        """
+        if not path.endswith(cls.binary_extensions):
+            raise ValueError(
+                f"Only path ending with {cls.binary_extensions} are supported."
+            )
+        path = Path(path.strip("/"))
+        class_name = path.name
+        class_name, _, _ = class_name.partition(".")  # plain .class
+        return str(path.parent / f"{class_name}{extension}")
+
 
 def find_expression(lines, regex):
     """
@@ -177,12 +192,39 @@ class JavaLanguage(JvmLanguage):
 class ScalaLanguage(JvmLanguage):
     name = "scala"
     source_extensions = (".scala",)
-    binary_extensions = (".class",)
+    binary_extensions = (".class", ".tasty")
     source_package_attribute_name = "scala_package"
     package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
     binary_map_type = "scala_to_class"
 
 
+class GroovyLanguage(JvmLanguage):
+    name = "groovy"
+    source_extensions = (".groovy",)
+    binary_extensions = (".class",)
+    source_package_attribute_name = "groovy_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "groovy_to_class"
+
+
+class AspectJLanguage(JvmLanguage):
+    name = "aspectj"
+    source_extensions = (".aj",)
+    binary_extensions = (".class",)
+    source_package_attribute_name = "aspectj_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "aspectj_to_class"
+
+
+class ClojureLanguage(JvmLanguage):
+    name = "clojure"
+    source_extensions = (".clj",)
+    binary_extensions = (".class",)
+    source_package_attribute_name = "clojure_package"
+    package_regex = re.compile(r"^\s*package\s+([\w\.]+)\s*;?")
+    binary_map_type = "clojure_to_class"
+
+
 class KotlinLanguage(JvmLanguage):
     name = "kotlin"
     source_extensions = (".kt", ".kts")