Add new license detection rules (#3575)

pombredanne · AyanSinhaMahapatra · web-flow · commit 1c4d6bedd7bd · 2023-11-09T20:08:14.000+05:30
* Add new license detection rules

Also add trailing LF to license rule texts

Signed-off-by: Philippe Ombredanne &lt;pombredanne@nexb.com&gt;
Signed-off-by: Ayan Sinha Mahapatra &lt;ayansmahapatra@gmail.com&gt;
Co-authored-by: Ayan Sinha Mahapatra &lt;ayansmahapatra@gmail.com&gt;
diff --git a/etc/scripts/licenses/buildrules.py b/etc/scripts/licenses/buildrules.py
@@ -61,7 +61,7 @@ class RuleData(object):
 
     def __attrs_post_init__(self, *args, **kwargs):
         self.raw_data = rdat = "\n".join(self.data_lines).strip()
-        self.text = "\n".join(self.text_lines).strip()
+        self.text = "\n".join(self.text_lines).strip() + "\n"
 
         # validate YAML syntax
         try:
diff --git a/src/licensedcode/data/rules/apache-2.0_1316.RULE b/src/licensedcode/data/rules/apache-2.0_1316.RULE
@@ -0,0 +1,7 @@
+---
+license_expression: apache-2.0
+is_license_tag: yes
+relevance: 100
+---
+
+name: The Apache Software License, Version 2.0
diff --git a/src/licensedcode/data/rules/apache-2.0_1317.RULE b/src/licensedcode/data/rules/apache-2.0_1317.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: apache-2.0
+is_license_tag: yes
+relevance: 100
+ignorable_urls:
+    - http://www.apache.org/license/LICENSE-2.0.txt
+---
+
+url: http://www.apache.org/license/LICENSE-2.0.txt
diff --git a/src/licensedcode/data/rules/apache-2.0_1318.RULE b/src/licensedcode/data/rules/apache-2.0_1318.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: apache-2.0
+is_license_reference: yes
+relevance: 100
+ignorable_urls:
+    - http://www.apache.org/license/LICENSE-2.0.txt
+---
+
+http://www.apache.org/license/LICENSE-2.0.txt
diff --git a/src/licensedcode/data/rules/authorizenet-sdk_3.RULE b/src/licensedcode/data/rules/authorizenet-sdk_3.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: authorizenet-sdk
+is_license_tag: yes
+ignorable_urls:
+    - https://github.com/AuthorizeNet/sdk-java/blob/master/LICENSE.txt
+---
+
+name: SDK License Agreement
+url: {{https://github.com/AuthorizeNet/sdk-java/blob/master/LICENSE.txt}}
diff --git a/src/licensedcode/data/rules/bsd-new_1346.RULE b/src/licensedcode/data/rules/bsd-new_1346.RULE
@@ -0,0 +1,10 @@
+---
+license_expression: bsd-new
+is_license_tag: yes
+ignorable_urls:
+    - http://www.linfo.org/bsdlicense.html
+---
+
+name: BSD
+url: http://www.linfo.org/bsdlicense.html
+comments: Free for open or commercial use, with credit line
diff --git a/src/licensedcode/data/rules/epl-2.0_76.RULE b/src/licensedcode/data/rules/epl-2.0_76.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: epl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
+---
+
+- name: Eclipse Public License v. 2.0
+  url: https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
diff --git a/src/licensedcode/data/rules/epl-2.0_77.RULE b/src/licensedcode/data/rules/epl-2.0_77.RULE
@@ -0,0 +1,10 @@
+---
+license_expression: epl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
+---
+
+license
+            <name>Eclipse Public License v. 2.0
+            <url>https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
diff --git a/src/licensedcode/data/rules/epl-2.0_or_gpl-2.0_with_classpath-exception-2.0_10.RULE b/src/licensedcode/data/rules/epl-2.0_or_gpl-2.0_with_classpath-exception-2.0_10.RULE
@@ -0,0 +1,12 @@
+---
+license_expression: epl-2.0 OR gpl-2.0 WITH classpath-exception-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
+    - https://www.gnu.org/software/classpath/license.html
+---
+
+<name>Eclipse Public License v. 2.0</name>
+            <url>https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt</url>
+            <name>GNU General Public License, version 2 with the GNU Classpath Exception</name>
+            <url>https://www.gnu.org/software/classpath/license.html</url>
diff --git a/src/licensedcode/data/rules/epl-2.0_or_gpl-2.0_with_classpath-exception-2.0_11.RULE b/src/licensedcode/data/rules/epl-2.0_or_gpl-2.0_with_classpath-exception-2.0_11.RULE
@@ -0,0 +1,13 @@
+---
+license_expression: epl-2.0 OR gpl-2.0 WITH classpath-exception-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
+    - https://www.gnu.org/software/classpath/license.html
+---
+
+license
+            <name>Eclipse Public License v. 2.0</name>
+            <url>https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt</url>
+            <name>GNU General Public License, version 2 with the GNU Classpath Exception</name>
+            <url>https://www.gnu.org/software/classpath/license.html</url>
diff --git a/src/licensedcode/data/rules/epl-2.0_or_gpl-2.0_with_classpath-exception-2.0_12.RULE b/src/licensedcode/data/rules/epl-2.0_or_gpl-2.0_with_classpath-exception-2.0_12.RULE
@@ -0,0 +1,13 @@
+---
+license_expression: epl-2.0 OR gpl-2.0 WITH classpath-exception-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
+    - https://www.gnu.org/software/classpath/license.html
+---
+
+license
+            <name>Eclipse Public License v. 2.0
+            <url>https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
+            <name>GNU General Public License, version 2 with the GNU Classpath Exception
+            <url>https://www.gnu.org/software/classpath/license.html
diff --git a/src/licensedcode/data/rules/gpl-2.0_with_classpath-exception-2.0_61.RULE b/src/licensedcode/data/rules/gpl-2.0_with_classpath-exception-2.0_61.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: gpl-2.0 WITH classpath-exception-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.gnu.org/software/classpath/license.html
+---
+
+- name: GNU General Public License, version 2 with the GNU Classpath Exception
+  url: https://www.gnu.org/software/classpath/license.html
diff --git a/src/licensedcode/data/rules/gpl-2.0_with_classpath-exception-2.0_62.RULE b/src/licensedcode/data/rules/gpl-2.0_with_classpath-exception-2.0_62.RULE
@@ -0,0 +1,10 @@
+---
+license_expression: gpl-2.0 WITH classpath-exception-2.0
+is_license_tag: yes
+ignorable_urls:
+    - https://www.gnu.org/software/classpath/license.html
+---
+
+license
+            <name>GNU General Public License, version 2 with the GNU Classpath Exception
+            <url>https://www.gnu.org/software/classpath/license.html
diff --git a/src/licensedcode/data/rules/lgpl-3.0_320.RULE b/src/licensedcode/data/rules/lgpl-3.0_320.RULE
@@ -0,0 +1,12 @@
+---
+license_expression: lgpl-3.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/lgpl.html
+---
+
+<license>
+            <name>GNU General Lesser Public License (LGPL) version 3.0</name>
+            <url>http://www.gnu.org/licenses/lgpl.html</url>
+            <distribution>repo</distribution>
+        </license>
diff --git a/src/licensedcode/data/rules/lgpl-3.0_321.RULE b/src/licensedcode/data/rules/lgpl-3.0_321.RULE
@@ -0,0 +1,9 @@
+---
+license_expression: lgpl-3.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/lgpl.html
+---
+
+- name: GNU General Lesser Public License (LGPL) version 3.0
+url: http://www.gnu.org/licenses/lgpl.html
diff --git a/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_1.RULE b/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_1.RULE
@@ -0,0 +1,13 @@
+---
+license_expression: lgpl-3.0 OR mpl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/lgpl.html
+    - http://www.mozilla.org/MPL/2.0/
+---
+
+license
+- name: GNU General Lesser Public License (LGPL) version 3.0
+url: http://www.gnu.org/licenses/lgpl.html
+- name: Mozilla Public License Version 2.0
+url: http://www.mozilla.org/MPL/2.0/
diff --git a/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_2.RULE b/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_2.RULE
@@ -0,0 +1,14 @@
+---
+license_expression: lgpl-3.0 OR mpl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/lgpl.html
+    - http://www.mozilla.org/MPL/2.0/
+---
+
+license
+- name: GNU General Lesser Public License (LGPL) version 3.0
+url: http://www.gnu.org/licenses/lgpl.html
+- name: Mozilla Public License Version 2.0
+url: http://www.mozilla.org/MPL/2.0/
+license
diff --git a/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_3.RULE b/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_3.RULE
@@ -0,0 +1,15 @@
+---
+license_expression: lgpl-3.0 OR mpl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/lgpl.html
+    - http://www.mozilla.org/MPL/2.0/
+---
+
+licenses
+license
+- name: GNU General Lesser Public License (LGPL) version 3.0
+url: http://www.gnu.org/licenses/lgpl.html
+- name: Mozilla Public License Version 2.0
+url: http://www.mozilla.org/MPL/2.0/
+license
diff --git a/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_4.RULE b/src/licensedcode/data/rules/lgpl-3.0_or_mpl-2.0_4.RULE
@@ -0,0 +1,18 @@
+---
+license_expression: lgpl-3.0 OR mpl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.gnu.org/licenses/lgpl.html
+    - http://www.mozilla.org/MPL/2.0/
+---
+
+<license>
+            <name>GNU General Lesser Public License (LGPL) version 3.0</name>
+            <url>http://www.gnu.org/licenses/lgpl.html</url>
+            <distribution>repo</distribution>
+        </license>
+        <license>
+            <name>Mozilla Public License Version 2.0</name>
+            <url>http://www.mozilla.org/MPL/2.0/</url>
+            <distribution>repo</distribution>
+        </license>
diff --git a/src/licensedcode/data/rules/license-clue_authorizenet-sdk_1.RULE b/src/licensedcode/data/rules/license-clue_authorizenet-sdk_1.RULE
@@ -0,0 +1,8 @@
+---
+license_expression: authorizenet-sdk
+is_license_clue: yes
+is_continuous: yes
+relevance: 70
+---
+
+{{AuthorizeNet sdk license}}
diff --git a/src/licensedcode/data/rules/mpl-2.0_139.RULE b/src/licensedcode/data/rules/mpl-2.0_139.RULE
@@ -0,0 +1,12 @@
+---
+license_expression: mpl-2.0
+is_license_tag: yes
+ignorable_urls:
+    - http://www.mozilla.org/MPL/2.0/
+---
+
+<license>
+            <name>Mozilla Public License Version 2.0</name>
+            <url>http://www.mozilla.org/MPL/2.0/</url>
+            <distribution>repo</distribution>
+        </license>
diff --git a/src/licensedcode/data/rules/mpl-2.0_140.RULE b/src/licensedcode/data/rules/mpl-2.0_140.RULE
@@ -0,0 +1,10 @@
+---
+license_expression: mpl-2.0
+is_license_tag: yes
+relevance: 100
+ignorable_urls:
+    - http://www.mozilla.org/MPL/2.0/
+---
+
+- name: Mozilla Public License Version 2.0
+url: http://www.mozilla.org/MPL/2.0/
diff --git a/src/licensedcode/data/rules/other-permissive_425.RULE b/src/licensedcode/data/rules/other-permissive_425.RULE
@@ -0,0 +1,16 @@
+---
+license_expression: other-permissive
+is_license_notice: yes
+minimum_coverage: 95
+notes: Seen in openpdf See also https://web.archive.org/web/20060113033212/http://www.javaworld.com/javaworld/javatips/jw-javatip128.html
+ignorable_urls:
+    - http://www.javaworld.com/javaworld/javatips/jw-javatip128.html
+---
+
+SimpleXMLParser:
+The original version of this class was published in a JavaWorld article by Steven Brandt:
+http://www.javaworld.com/javaworld/javatips/jw-javatip128.html
+Jennifer Orr (JavaWorld) wrote: "You have permission to use the code appearing in
+Steven Brandt's JavaWorld article, 'Java Tip 128: Create a quick-and-dirty XML parser.'
+We ask that you reference the author as the creator and JavaWorld as the original publisher
+of the code." Steven Brandt also agreed with the use of this class.
diff --git a/src/licensedcode/data/rules/other-permissive_426.RULE b/src/licensedcode/data/rules/other-permissive_426.RULE
@@ -0,0 +1,18 @@
+---
+license_expression: other-permissive
+is_license_text: yes
+notes: Similar to hpnd-sell-variant-mit-disclaimer and seen in https://gitlab.freedesktop.org/xorg/xserver/-/blob/141e7dd8a373eafe28fb4f6433671979fb4c59a7/COPYING#L1781
+---
+
+Permission to use, copy, modify, distribute, and sell this software and its
+documentation for any purpose is hereby granted without fee, provided that
+this permission notice appear in supporting documentation.  This permission
+notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/tests/licensedcode/data/datadriven/lic4/sun-bsd-no-nuclear_and_x11-tiff_and_unicode.txt.yml b/tests/licensedcode/data/datadriven/lic4/sun-bsd-no-nuclear_and_x11-tiff_and_unicode.txt.yml
@@ -1,4 +1,5 @@
 license_expressions:
+  - other-permissive
   - bsd-3-clause-no-nuclear-warranty
   - bsd-3-clause-no-nuclear-warranty
   - x11-tiff
diff --git a/tests/packagedcode/data/m2/codec/commons-codec/1.3/commons-codec-1.3.pom.package.json b/tests/packagedcode/data/m2/codec/commons-codec/1.3/commons-codec-1.3.pom.package.json
@@ -159,17 +159,17 @@
           "score": 100.0,
           "start_line": 1,
           "end_line": 1,
-          "matched_length": 7,
+          "matched_length": 8,
           "match_coverage": 100.0,
           "matcher": "2-aho",
           "license_expression": "apache-2.0",
-          "rule_identifier": "apache-2.0_5.RULE",
+          "rule_identifier": "apache-2.0_1316.RULE",
           "rule_relevance": 100,
-          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_5.RULE",
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_1316.RULE",
           "matched_text": "- name: The Apache Software License, Version 2.0"
         }
       ],
-      "identifier": "apache_2_0-daaaee76-1395-c8c8-e06a-3f3f76e079b1"
+      "identifier": "apache_2_0-c3b9cbd8-09fc-8aa5-f48b-917b9001c088"
     }
   ],
   "other_license_expression": null,
diff --git a/tests/packagedcode/data/m2/collections/commons-collections/3.2/commons-collections-3.2.pom.package.json b/tests/packagedcode/data/m2/collections/commons-collections/3.2/commons-collections-3.2.pom.package.json
@@ -698,17 +698,17 @@
           "score": 100.0,
           "start_line": 1,
           "end_line": 1,
-          "matched_length": 7,
+          "matched_length": 8,
           "match_coverage": 100.0,
           "matcher": "2-aho",
           "license_expression": "apache-2.0",
-          "rule_identifier": "apache-2.0_5.RULE",
+          "rule_identifier": "apache-2.0_1316.RULE",
           "rule_relevance": 100,
-          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_5.RULE",
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_1316.RULE",
           "matched_text": "- name: The Apache Software License, Version 2.0"
         }
       ],
-      "identifier": "apache_2_0-daaaee76-1395-c8c8-e06a-3f3f76e079b1"
+      "identifier": "apache_2_0-c3b9cbd8-09fc-8aa5-f48b-917b9001c088"
     }
   ],
   "other_license_expression": null,
diff --git a/tests/packagedcode/data/m2/digester/commons-digester/1.8/commons-digester-1.8.pom.package.json b/tests/packagedcode/data/m2/digester/commons-digester/1.8/commons-digester-1.8.pom.package.json
@@ -194,17 +194,17 @@
           "score": 100.0,
           "start_line": 1,
           "end_line": 1,
-          "matched_length": 7,
+          "matched_length": 8,
           "match_coverage": 100.0,
           "matcher": "2-aho",
           "license_expression": "apache-2.0",
-          "rule_identifier": "apache-2.0_5.RULE",
+          "rule_identifier": "apache-2.0_1316.RULE",
           "rule_relevance": 100,
-          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_5.RULE",
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_1316.RULE",
           "matched_text": "- name: The Apache Software License, Version 2.0"
         }
       ],
-      "identifier": "apache_2_0-daaaee76-1395-c8c8-e06a-3f3f76e079b1"
+      "identifier": "apache_2_0-c3b9cbd8-09fc-8aa5-f48b-917b9001c088"
     }
   ],
   "other_license_expression": null,
diff --git a/tests/packagedcode/data/m2/el/commons-el/1.0/commons-el-1.0.pom.package.json b/tests/packagedcode/data/m2/el/commons-el/1.0/commons-el-1.0.pom.package.json
diff --git a/tests/packagedcode/data/m2/io/commons-io/1.3.1/commons-io-1.3.1.pom.package.json b/tests/packagedcode/data/m2/io/commons-io/1.3.1/commons-io-1.3.1.pom.package.json
diff --git a/tests/packagedcode/data/m2/lang/commons-lang/2.1/commons-lang-2.1.pom.package.json b/tests/packagedcode/data/m2/lang/commons-lang/2.1/commons-lang-2.1.pom.package.json
diff --git a/tests/packagedcode/data/m2/lang/commons-lang/2.3/commons-lang-2.3.pom.package.json b/tests/packagedcode/data/m2/lang/commons-lang/2.3/commons-lang-2.3.pom.package.json
diff --git a/tests/packagedcode/data/m2/logging/commons-logging/1.0.4/commons-logging-1.0.4.pom.package.json b/tests/packagedcode/data/m2/logging/commons-logging/1.0.4/commons-logging-1.0.4.pom.package.json
diff --git a/tests/packagedcode/data/m2/logging/commons-logging/1.1/commons-logging-1.1.pom.package.json b/tests/packagedcode/data/m2/logging/commons-logging/1.1/commons-logging-1.1.pom.package.json
diff --git a/tests/packagedcode/data/m2/net/sf/ehcache/ehcache/1.2.3/ehcache-1.2.3.pom.package.json b/tests/packagedcode/data/m2/net/sf/ehcache/ehcache/1.2.3/ehcache-1.2.3.pom.package.json
diff --git a/tests/packagedcode/data/m2/validator/commons-validator/1.2.0/commons-validator-1.2.0.pom.package.json b/tests/packagedcode/data/m2/validator/commons-validator/1.2.0/commons-validator-1.2.0.pom.package.json
diff --git a/tests/packagedcode/data/m2/validator/commons-validator/1.3.1/commons-validator-1.3.1.pom.package.json b/tests/packagedcode/data/m2/validator/commons-validator/1.3.1/commons-validator-1.3.1.pom.package.json
diff --git a/tests/packagedcode/data/maven2/commons-validator-1.2.0/commons-validator-1.2.0.pom.package.json b/tests/packagedcode/data/maven2/commons-validator-1.2.0/commons-validator-1.2.0.pom.package.json
diff --git a/tests/packagedcode/data/plugin/maven-package-expected.json b/tests/packagedcode/data/plugin/maven-package-expected.json
diff --git a/tests/packagedcode/data/plugin/maven-package-with-license-expected.json b/tests/packagedcode/data/plugin/maven-package-with-license-expected.json
