Merge remote-tracking branch 'origin/main' into feature/1949-d2d-multiple-binaries

merging & resolving conflicts

Author: rejzzzz

README.rst

@@ -57,9 +57,8 @@ for any legal advice.
 
 
 
-
-.. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml/badge.svg?branch=main
-    :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml
+.. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/run-unit-tests.yml/badge.svg?branch=main
+    :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/run-unit-tests.yml
     :alt: CI Tests Status
 
 .. |docs-rtd| image:: https://readthedocs.org/projects/scancodeio/badge/?version=latest

pyproject.toml

@@ -123,7 +123,7 @@ android_analysis = [
   "android_inspector==0.0.1"
 ]
 mining = [
-  "minecode_pipelines==0.0.1b8"
+  "minecode_pipelines==0.1.1"
 ]
 
 [project.urls]

scanpipe/pipes/cyclonedx.py

@@ -30,6 +30,7 @@
 from cyclonedx.model import license as cdx_license_model
 from cyclonedx.model.bom import Bom
 from cyclonedx.schema import SchemaVersion
+from cyclonedx.schema.schema import BaseSchemaVersion
 from cyclonedx.validation import ValidationError
 from cyclonedx.validation.json import JsonStrictValidator
 from defusedxml import ElementTree as SafeElementTree
@@ -184,10 +185,12 @@ def cyclonedx_component_to_package_data(
     affected_by_vulnerabilities = []
     if affected_by := vulnerabilities.get(bom_ref):
         for cdx_vulnerability in affected_by:
+            cdx_vulnerability_json = cdx_vulnerability.as_json(view_=BaseSchemaVersion)
             affected_by_vulnerabilities.append(
                 {
                     "vulnerability_id": str(cdx_vulnerability.id),
                     "summary": cdx_vulnerability.description,
+                    "cdx_vulnerability_data": json.loads(cdx_vulnerability_json),
                 }
             )
 

scanpipe/pipes/d2d.py

@@ -174,12 +174,21 @@ def _map_jvm_to_class_resource(
     from/ fully qualified binary files.
     """
     for extension in jvm_lang.source_extensions:
-        normalized_path = jvm_lang.get_normalized_path(
+        # Perform basic conversion from .class to source file path
+        source_path = jvm_lang.get_source_path(
             path=to_resource.path, extension=extension
         )
-        match = pathmap.find_paths(path=normalized_path, index=from_classes_index)
+        # Perform basic mapping without normalization for scenarios listed in
+        # https://github.com/aboutcode-org/scancode.io/issues/1873
+        match = pathmap.find_paths(path=source_path, index=from_classes_index)
+
         if not match:
-            return
+            normalized_path = jvm_lang.get_normalized_path(
+                path=to_resource.path, extension=extension
+            )
+            match = pathmap.find_paths(path=normalized_path, index=from_classes_index)
+            if not match:
+                return
 
         for resource_id in match.resource_ids:
             from_resource = from_resources.get(id=resource_id)

scanpipe/pipes/jvm.py

@@ -140,12 +140,27 @@ def get_normalized_path(cls, path, extension):
         # https://github.com/aboutcode-org/scancode.io/issues/1994
         if class_name.endswith("_$logger.class"):
             class_name, _, _ = class_name.partition("_$logger.class")
-        elif "$" in class_name:  # inner class
+        elif "$" in class_name and not class_name.startswith("$"):  # inner class
             class_name, _, _ = class_name.partition("$")
         else:
             class_name, _, _ = class_name.partition(".")  # plain .class
         return str(path.parent / f"{class_name}{extension}")
 
+    @classmethod
+    def get_source_path(cls, path, extension):
+        """
+        Return a JVM file path for ``path`` .class file path string.
+        No normalization is performed.
+        """
+        if not path.endswith(cls.binary_extensions):
+            raise ValueError(
+                f"Only path ending with {cls.binary_extensions} are supported."
+            )
+        path = Path(path.strip("/"))
+        class_name = path.name
+        class_name, _, _ = class_name.partition(".")  # plain .class
+        return str(path.parent / f"{class_name}{extension}")
+
 
 def find_expression(lines, regex):
     """

scanpipe/pipes/ort.py

@@ -130,14 +130,28 @@ def get_ort_project_type(project):
         return "docker"
 
 
+def sanitize_id_part(value):
+    """
+    Sanitize an identifier part by replacing colons with underscores.
+    ORT uses colons as separators in the identifier string representation.
+    """
+    if value:
+        return value.replace(":", "_")
+    return value
+
+
 def to_ort_package_list_yml(project):
     """Convert a project object into a YAML string in the ORT package list format."""
     project_type = get_ort_project_type(project)
 
     dependencies = []
     for package in project.discoveredpackages.all():
+        type_ = sanitize_id_part(project_type or package.type)
+        name = sanitize_id_part(package.name)
+        version = sanitize_id_part(package.version)
+
         dependency = Dependency(
-            id=f"{project_type or package.type}::{package.name}:{package.version}",
+            id=f"{type_}::{name}:{version}",
             purl=package.purl,
             sourceArtifact=SourceArtifact(url=package.download_url),
             declaredLicenses=[package.get_declared_license_expression_spdx()],

scanpipe/pipes/pathmap.py

@@ -164,11 +164,12 @@ def get_reversed_path_segments(path):
     Return reversed segments list given a POSIX ``path`` string. We reverse
     based on path segments separated by a "/".
 
-    Note that the inputh ``path`` is assumed to be normalized, not relative and
+    Note that the input ``path`` is assumed to be normalized, not relative and
     not containing double slash.
 
-    For example::
-    >>> assert get_reversed_path_segments("a/b/c.js") == ["c.js", "b", "a"]
+    For example:
+    >>> get_reversed_path_segments("a/b/c.js")
+    ['c.js', 'b', 'a']
     """
     # [::-1] does the list reversing
     reversed_segments = path.strip("/").split("/")[::-1]
@@ -177,13 +178,14 @@ def get_reversed_path_segments(path):
 
 def convert_segments_to_path(segments):
     """
-    Return a path string is suitable for indexing or matching given a
+    Return a path string suitable for indexing or matching given a
     ``segments`` sequence of path segment strings.
     The resulting reversed path is prefixed and suffixed by a "/" irrespective
     of whether the original path is a file or directory and had such prefix or
     suffix.
 
-    For example::
-    >>> assert convert_segments_to_path(["c.js", "b", "a"]) == "/c.js/b/a/"
+    For example:
+    >>> convert_segments_to_path(["c.js", "b", "a"])
+    '/c.js/b/a/'
     """
     return "/" + "/".join(segments) + "/"

scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html

@@ -2,9 +2,9 @@
   <table class="table is-bordered is-striped is-narrow is-hoverable is-fullwidth">
     <thead>
       <tr>
-        <th style="width: 210px;">Affected by</th>
+        <th style="width: 220px;">Affected by</th>
         <th>Summary</th>
-        <th style="width: 210px;">Aliases</th>
+        <th>Analysis</th>
       </tr>
     </thead>
     <tbody>
@@ -15,28 +15,43 @@
               {{ vulnerability.vulnerability_id }}
               <i class="fa-solid fa-up-right-from-square is-small"></i>
             </a>
+            <ul class="list-unstyled mb-0">
+              {% for alias in aliases %}
+                <li>
+                  {% if alias|slice:":3" == "CVE" %}
+                    <a href="https://nvd.nist.gov/vuln/detail/{{ alias }}" target="_blank">{{ alias }}
+                      <i class="fa-solid fa-up-right-from-square mini"></i>
+                    </a>
+                  {% elif alias|slice:":4" == "GHSA" %}
+                    <a href="https://github.com/advisories/{{ alias }}" target="_blank">{{ alias }}
+                      <i class="fa-solid fa-up-right-from-square mini"></i>
+                    </a>
+                  {% elif alias|slice:":3" == "NPM" %}
+                    <a href="https://github.com/nodejs/security-wg/blob/main/vuln/npm/{{ alias|slice:"4:" }}.json" target="_blank">{{ alias }}
+                      <i class="fa-solid fa-up-right-from-square mini"></i>
+                    </a>
+                  {% else %}
+                    {{ alias }}
+                  {% endif %}
+                </li>
+              {% endfor %}
+            </ul>
           </td>
           <td>
-            {{ vulnerability.summary }}
-          </td>
-          <td>
-            {% for alias in vulnerability.aliases %}
-              {% if alias|slice:":3" == "CVE" %}
-                <a href="https://nvd.nist.gov/vuln/detail/{{ alias }}" target="_blank">{{ alias }}
-                  <i class="fa-solid fa-up-right-from-square is-small"></i>
-                </a>
-              {% elif alias|slice:":4" == "GHSA" %}
-                <a href="https://github.com/advisories/{{ alias }}" target="_blank">{{ alias }}
-                  <i class="fa-solid fa-up-right-from-square is-small"></i>
-                </a>
-              {% elif alias|slice:":3" == "NPM" %}
-                <a href="https://github.com/nodejs/security-wg/blob/main/vuln/npm/{{ alias|slice:"4:" }}.json" target="_blank">{{ alias }}
-                  <i class="fa-solid fa-up-right-from-square is-small"></i>
-                </a>
+            {% if vulnerability.summary %}
+              {% if vulnerability.summary|length > 150 %}
+                <details>
+                  <summary>{{ vulnerability.summary|slice:":150" }}...</summary>
+                  {{ vulnerability.summary|slice:"150:" }}
+                </details>
               {% else %}
-                {{ alias }}
+                {{ vulnerability.summary }}
               {% endif %}
-              <br>
+            {% endif %}
+          </td>
+          <td>
+            {% for key, value in vulnerability.cdx_vulnerability.analysis.items %}
+              <strong>{{ key }}:</strong> {{ value }}{% if not forloop.last %}<br>{% endif %}
             {% endfor %}
           </td>
         </tr>

scanpipe/tests/pipes/test_cyclonedx.py

@@ -250,13 +250,24 @@ def test_scanpipe_cyclonedx_resolve_cyclonedx_packages_vulnerabilities(self):
         self.assertEqual(1, len(packages))
 
         affected_by = packages[0]["affected_by_vulnerabilities"]
+        self.assertEqual("CVE-2005-2541", affected_by[0]["vulnerability_id"])
+        self.assertEqual(
+            "Tar 1.15.1 does not properly warn the user when...",
+            affected_by[0]["summary"],
+        )
+        self.assertIn("cdx_vulnerability_data", affected_by[0])
+        cdx_vulnerability_data = affected_by[0]["cdx_vulnerability_data"]
         expected = [
-            {
-                "vulnerability_id": "CVE-2005-2541",
-                "summary": "Tar 1.15.1 does not properly warn the user when...",
-            }
+            "advisories",
+            "affects",
+            "description",
+            "id",
+            "published",
+            "ratings",
+            "source",
+            "updated",
         ]
-        self.assertEqual(expected, affected_by)
+        self.assertEqual(expected, sorted(cdx_vulnerability_data.keys()))
 
     def test_scanpipe_cyclonedx_resolve_cyclonedx_packages_pre_validation(self):
         # This SBOM includes multiple deserialization issues that are "fixed"

scanpipe/tests/pipes/test_jvm.py

@@ -84,6 +84,16 @@ def test_scanpipe_pipes_jvm_get_java_package_too_far_down(self):
         package = jvm.JavaLanguage.get_source_package(input_location)
         self.assertIsNone(package)
 
+    def test_scanpipe_pipes_jvm_get_source_java_path(self):
+        njp = jvm.JavaLanguage.get_source_path("foo/org/common/Bar.class", ".java")
+        self.assertEqual("foo/org/common/Bar.java", njp)
+
+    def test_scanpipe_pipes_jvm_get_source_java_path_with_inner_class(self):
+        njp = jvm.JavaLanguage.get_source_path(
+            "foo/org/common/Bar$inner.class", ".java"
+        )
+        self.assertEqual("foo/org/common/Bar$inner.java", njp)
+
     def test_scanpipe_pipes_jvm_get_normalized_java_path(self):
         njp = jvm.JavaLanguage.get_normalized_path("foo/org/common/Bar.class", ".java")
         self.assertEqual("foo/org/common/Bar.java", njp)