Add test for javascript string match

Signed-off-by: Keshav Priyadarshi <git@keshav.space>

Author: keshav-space

scanpipe/pipes/d2d.py

@@ -59,7 +59,7 @@
 from scanpipe.pipes import purldb
 from scanpipe.pipes import resolve
 from scanpipe.pipes import scancode
-from scanpipe.pipes import strings
+from scanpipe.pipes import stringmap
 from scanpipe.pipes import symbolmap
 from scanpipe.pipes import symbols
 
@@ -2107,18 +2107,20 @@ def _map_javascript_strings(to_resource, javascript_from_resources, logger):
     """
     ignoreable_string_threshold = 5
     to_strings = to_resource.extra_data.get("source_strings")
+    to_strings_set = set(to_strings)
 
-    if not to_strings and len(to_strings) > ignoreable_string_threshold:
+    if not to_strings or len(to_strings_set) < ignoreable_string_threshold:
         return 0
 
     best_matching_score = 0
     best_match = None
     for source_js in javascript_from_resources:
         from_strings = source_js.extra_data.get("source_strings")
-        if not from_strings and len(from_strings) > ignoreable_string_threshold:
+        from_strings_set = set(from_strings)
+        if not from_strings or len(from_strings_set) < ignoreable_string_threshold:
             continue
 
-        is_match, similarity = strings.match_source_strings_to_deployed(
+        is_match, similarity = stringmap.match_source_strings_to_deployed(
             source_strings=from_strings,
             deployed_strings=to_strings,
         )

scanpipe/pipes/stringmap.py

@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from collections import Counter
+
+STRING_MATCHING_RATIO_JAVASCRIPT = 0.7
+SMALL_FILE_STRING_THRESHOLD_JAVASCRIPT = 10
+STRING_MATCHING_RATIO_JAVASCRIPT_SMALL_FILE = 0.5
+
+
+def match_source_strings_to_deployed(source_strings, deployed_strings):
+    """
+    Compute the similarity between source and deployed string literals and
+    return whether they match based on matching threshold.
+    """
+    common_strings_ratio = 0
+    is_match = False
+    deployed_strings_set = set(deployed_strings)
+    source_strings_set = set(source_strings)
+    source_strings_count = len(source_strings)
+    deployed_strings_count = len(deployed_strings)
+    total_strings_count = source_strings_count + deployed_strings_count
+    source_strings_counter = Counter(source_strings)
+    deployed_strings_counter = Counter(deployed_strings)
+
+    common_strings = source_strings_set.intersection(deployed_strings_set)
+    total_common_strings_count = sum(
+        [
+            source_strings_counter.get(string, 0)
+            + deployed_strings_counter.get(string, 0)
+            for string in common_strings
+        ]
+    )
+
+    if total_common_strings_count:
+        common_strings_ratio = total_common_strings_count / total_strings_count
+
+    if common_strings_ratio > STRING_MATCHING_RATIO_JAVASCRIPT:
+        is_match = True
+    elif (
+        source_strings_count > SMALL_FILE_STRING_THRESHOLD_JAVASCRIPT
+        and common_strings_ratio > STRING_MATCHING_RATIO_JAVASCRIPT_SMALL_FILE
+    ):
+        is_match = True
+
+    return is_match, common_strings_ratio

scanpipe/pipes/strings.py

@@ -20,14 +20,8 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-from collections import Counter
-
 from aboutcode.pipeline import LoopProgress
 
-STRING_MATCHING_RATIO_JAVASCRIPT = 0.7
-SMALL_FILE_STRING_THRESHOLD_JAVASCRIPT = 10
-STRING_MATCHING_RATIO_JAVASCRIPT_SMALL_FILE = 0.5
-
 
 class XgettextNotFound(Exception):
     pass
@@ -72,37 +66,3 @@ def _collect_and_store_resource_strings(resource):
     result = strings_xgettext.collect_strings(resource.location)
     strings = [item["string"] for item in result if "string" in item]
     resource.update_extra_data({"source_strings": strings})
-
-
-def match_source_strings_to_deployed(source_strings, deployed_strings):
-    common_strings_ratio = 0
-    is_match = False
-    deployed_strings_set = set(deployed_strings)
-    source_strings_set = set(source_strings)
-    source_strings_count = len(source_strings)
-    deployed_strings_count = len(deployed_strings)
-    total_strings_count = source_strings_count + deployed_strings_count
-    source_strings_counter = Counter(source_strings)
-    deployed_strings_counter = Counter(deployed_strings)
-
-    common_strings = source_strings_set.intersection(deployed_strings_set)
-    total_common_strings_count = sum(
-        [
-            source_strings_counter.get(string, 0)
-            + deployed_strings_counter.get(string, 0)
-            for string in common_strings
-        ]
-    )
-
-    if total_common_strings_count:
-        common_strings_ratio = total_common_strings_count / total_strings_count
-
-    if common_strings_ratio > STRING_MATCHING_RATIO_JAVASCRIPT:
-        is_match = True
-    elif (
-        source_strings_count > SMALL_FILE_STRING_THRESHOLD_JAVASCRIPT
-        and common_strings_ratio > STRING_MATCHING_RATIO_JAVASCRIPT_SMALL_FILE
-    ):
-        is_match = True
-
-    return is_match, common_strings_ratio

scanpipe/tests/data/d2d-javascript/strings/cesium/cesium.ABOUT

@@ -0,0 +1,8 @@
+about_resource: cesium
+name: cesium
+version: 1.125
+download_url: https://github.com/CesiumGS/cesium/archive/refs/tags/1.125.zip
+homepage_url: https://github.com/CesiumGS/cesium
+license_expression: apache-2.0
+attribute: yes
+package_url: pkg:github/CesiumGS/cesium@1.125