Merge branch 'main' into python-d2d

Author: JonoYang

CHANGELOG.rst

@@ -10,6 +10,12 @@ v35.1.0 (unreleased)
   Requires the ``find_vulnerabilities`` pipeline to be executed beforehand.
   https://github.com/aboutcode-org/scancode.io/pull/1702
 
+- Enable ``--license-references`` scan option in the ``scan_single_package`` pipeline.
+  The ``license_references`` and ``license_rule_references`` attributes will now be
+  available in the scan results, including the details about detected licenses and
+  license rules used during the scan.
+  https://github.com/aboutcode-org/scancode.io/issues/1657
+
 - Add a new step to the ``DeployToDevelop`` pipeline, ``match_python``, to match
   Cython source files (.pyx) to their compiled binaries.
   https://github.com/aboutcode-org/scancode.io/pull/1703

scancodeio/static/add-inputs.js

@@ -21,15 +21,41 @@
 // Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 const fileInput = document.querySelector("#id_input_files");
+let selectedFiles = []; // Store selected files
 fileInput.onchange = updateFiles;
 
 // Update the list of files to be uploaded in the UI
 function updateFiles() {
   if (fileInput.files.length > 0) {
     const fileName = document.querySelector("#inputs_file_name");
     fileName.innerHTML = "";
-    for (let file of fileInput.files) {
-      fileName.innerHTML += `<span class="is-block">${file.name}</span>`;
+
+    // Update the selectedFiles array
+    const newFiles = Array.from(fileInput.files);
+    // Create a Set to track unique file names
+    const uniqueFileNames = new Set(selectedFiles.map(file => file.name));
+    // Filter out files with the same name
+    const filteredNewFiles = newFiles.filter(file => !uniqueFileNames.has(file.name));
+    // Concatenate the unique files to the existing selectedFiles array
+    selectedFiles = selectedFiles.concat(filteredNewFiles);
+
+    for (let file of selectedFiles) {
+      const fileNameWithoutSpaces = file.name.replace(/\s/g, '');
+      fileName.innerHTML += `
+      <span class="is-flex is-justify-content-space-between is-block" id="file-name-${fileNameWithoutSpaces}">
+        <span class="is-block">${file.name}</span>
+        <a href="#" onclick="removeFile('${fileNameWithoutSpaces}')" class="model-button" id="file-delete-btn-${fileNameWithoutSpaces}">
+          <i class="fa-solid fa-trash-can"></i>
+        </a>
+      </span>
+      `;
+      document.getElementById("file-delete-btn-"+ fileNameWithoutSpaces).addEventListener("click", function(event){
+        disableEvent(event);
+        removeFile(fileNameWithoutSpaces);
+        if(selectedFiles.length == 0){
+          fileName.innerHTML ="<i>No files selected</i>"
+        }
+      });
     }
   }
 }
@@ -40,15 +66,37 @@ function disableEvent(event) {
   event.preventDefault();
 }
 
+function removeFile(fileName) {
+  selectedFiles = selectedFiles.filter(file => {
+    const fileNameWithoutSpaces = file.name.replace(/\s/g, '');
+    return fileNameWithoutSpaces !== fileName;
+  });
+
+  const fileNameElement = document.getElementById(`file-name-${fileName}`);
+  if (fileNameElement) {
+    fileNameElement.remove();
+  }
+
+  const dataTransfer = new DataTransfer();
+  for (let file of selectedFiles) {
+    dataTransfer.items.add(file);
+  }
+
+  fileInput.files = dataTransfer.files;
+}
+
 function dropHandler(event) {
   disableEvent(event);
   const droppedFiles = event.dataTransfer.files;
-  const updatedFiles = Array.from(fileInput.files);
+  const updatedFilesSet = new Set(Array.from(fileInput.files));
 
   for (let file of droppedFiles) {
-    updatedFiles.push(file);
+    updatedFilesSet.add(file);
   }
 
+  // Convert the Set back to an array if needed
+  const updatedFiles = Array.from(updatedFilesSet);
+
   const dataTransfer = new DataTransfer();
   for (let file of updatedFiles) {
     dataTransfer.items.add(file);

scanpipe/filters.py

@@ -483,6 +483,7 @@ def filter(self, qs, value):
     ("about_file", "about file"),
     ("java_to_class", "java to class"),
     ("jar_to_source", "jar to source"),
+    ("javascript_strings", "js strings"),
     ("javascript_symbols", "js symbols"),
     ("js_compiled", "js compiled"),
     ("js_colocation", "js colocation"),

scanpipe/pipelines/__init__.py

@@ -78,7 +78,10 @@ def flag_ignored_resources(self):
             ignored_patterns = ignored_patterns.splitlines()
         ignored_patterns.extend(flag.DEFAULT_IGNORED_PATTERNS)
 
-        flag.flag_ignored_patterns(self.project, patterns=ignored_patterns)
+        flag.flag_ignored_patterns(
+            codebaseresources=self.project.codebaseresources.no_status(),
+            patterns=ignored_patterns,
+        )
 
     def extract_archive(self, location, target):
         """Extract archive at `location` to `target`. Save errors as messages."""
@@ -179,6 +182,8 @@ def __init__(self, run_instance):
         self.selected_groups = run_instance.selected_groups
         self.selected_steps = run_instance.selected_steps
 
+        self.ecosystem_config = None
+
     @classmethod
     def get_initial_steps(cls):
         """Add the ``download_inputs`` step as an initial step if enabled."""

scanpipe/pipelines/deploy_to_develop.py

@@ -24,6 +24,7 @@
 from scanpipe import pipes
 from scanpipe.pipelines import Pipeline
 from scanpipe.pipes import d2d
+from scanpipe.pipes import d2d_config
 from scanpipe.pipes import flag
 from scanpipe.pipes import input
 from scanpipe.pipes import matchcode
@@ -64,6 +65,8 @@ def steps(cls):
             cls.flag_empty_files,
             cls.flag_whitespace_files,
             cls.flag_ignored_resources,
+            cls.load_ecosystem_config,
+            cls.map_ruby,
             cls.map_about_files,
             cls.map_checksum,
             cls.match_archives_to_purldb,
@@ -72,6 +75,7 @@ def steps(cls):
             cls.map_jar_to_source,
             cls.map_javascript,
             cls.map_javascript_symbols,
+            cls.map_javascript_strings,
             cls.map_elf,
             cls.map_macho,
             cls.map_winpe,
@@ -95,33 +99,6 @@ def steps(cls):
             cls.create_local_files_packages,
         )
 
-    purldb_package_extensions = [".jar", ".war", ".zip"]
-    purldb_resource_extensions = [
-        ".map",
-        ".js",
-        ".mjs",
-        ".ts",
-        ".d.ts",
-        ".jsx",
-        ".tsx",
-        ".css",
-        ".scss",
-        ".less",
-        ".sass",
-        ".soy",
-        ".class",
-    ]
-    doc_extensions = [
-        ".pdf",
-        ".doc",
-        ".docx",
-        ".ppt",
-        ".pptx",
-        ".tex",
-        ".odt",
-        ".odp",
-    ]
-
     def get_inputs(self):
         """Locate the ``from`` and ``to`` input files."""
         self.from_files, self.to_files = d2d.get_inputs(self.project)
@@ -156,6 +133,15 @@ def flag_whitespace_files(self):
         """Flag whitespace files with size less than or equal to 100 byte as ignored."""
         d2d.flag_whitespace_files(project=self.project)
 
+    def load_ecosystem_config(self):
+        """Load ecosystem specific configurations for d2d steps for selected options."""
+        d2d_config.load_ecosystem_config(pipeline=self, options=self.selected_groups)
+
+    @optional_step("Ruby")
+    def map_ruby(self):
+        """Load Ruby specific configurations for d2d steps."""
+        pass
+
     def map_about_files(self):
         """Map ``from/`` .ABOUT files to their related ``to/`` resources."""
         d2d.map_about_files(project=self.project, logger=self.log)
@@ -172,7 +158,7 @@ def match_archives_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_package_extensions,
+            extensions=self.matchable_package_extensions,
             matcher_func=d2d.match_purldb_package,
             logger=self.log,
         )
@@ -205,6 +191,11 @@ def map_javascript_symbols(self):
         """Map deployed JavaScript, TypeScript to its sources using symbols."""
         d2d.map_javascript_symbols(project=self.project, logger=self.log)
 
+    @optional_step("JavaScript")
+    def map_javascript_strings(self):
+        """Map deployed JavaScript, TypeScript to its sources using string literals."""
+        d2d.map_javascript_strings(project=self.project, logger=self.log)
+
     @optional_step("Elf")
     def map_elf(self):
         """Map ELF binaries to their sources using dwarf paths and symbols."""
@@ -258,7 +249,7 @@ def match_resources_to_purldb(self):
 
         d2d.match_purldb_resources(
             project=self.project,
-            extensions=self.purldb_resource_extensions,
+            extensions=self.matchable_resource_extensions,
             matcher_func=d2d.match_purldb_resource,
             logger=self.log,
         )
@@ -296,6 +287,7 @@ def flag_mapped_resources_archives_and_ignored_directories(self):
     def perform_house_keeping_tasks(self):
         """
         On deployed side
+            - Ignore specific files based on ecosystem based configurations.
             - PurlDB match files with ``no-java-source`` and empty status,
                 if no match is found update status to ``requires-review``.
             - Update status for uninteresting files.
@@ -306,9 +298,14 @@ def perform_house_keeping_tasks(self):
         """
         d2d.match_resources_with_no_java_source(project=self.project, logger=self.log)
         d2d.handle_dangling_deployed_legal_files(project=self.project, logger=self.log)
+        d2d.ignore_unmapped_resources_from_config(
+            project=self.project,
+            patterns_to_ignore=self.ecosystem_config.deployed_resource_path_exclusions,
+            logger=self.log,
+        )
         d2d.match_unmapped_resources(
             project=self.project,
-            matched_extensions=self.purldb_resource_extensions,
+            matched_extensions=self.ecosystem_config.matchable_resource_extensions,
             logger=self.log,
         )
         d2d.flag_undeployed_resources(project=self.project)
@@ -344,5 +341,5 @@ def flag_deployed_from_resources_with_missing_license(self):
         """Update the status for deployed from files with missing license."""
         d2d.flag_deployed_from_resources_with_missing_license(
             self.project,
-            doc_extensions=self.doc_extensions,
+            doc_extensions=self.ecosystem_config.doc_extensions,
         )

scanpipe/pipelines/scan_single_package.py

@@ -61,6 +61,7 @@ def steps(cls):
         "info": True,
         "license": True,
         "license_text": True,
+        "license_references": True,
         "package": True,
         "url": True,
         "classify": True,