extract: Prohibit specific files from being extracted

* Known DMCA risks.

Change-Id: I64d683b49b27537b4b87af80840561da7197a6dc

Author: npjohnson

extract_utils/args.py

@@ -88,6 +88,11 @@
     '--download-sha256',
     help='SHA256 of the download',
 )
+parser.add_argument(
+    '--allow-prohibited-files',
+    action='store_true',
+    help='Allow extraction of normally-prohibited files',
+)
 
 parser.add_argument(
     'source',
@@ -120,6 +125,7 @@ def __init__(self, args: argparse.Namespace):
         self.section: Optional[str] = args.section
         self.download_dir: Optional[str] = args.download_dir
         self.download_sha256: Optional[str] = args.download_sha256
+        self.allow_prohibited_files: bool = args.allow_prohibited_files
 
         if self.download_dir is None and DOWNLOAD_DIR_ENV_KEY in os.environ:
             self.download_dir = os.environ[DOWNLOAD_DIR_ENV_KEY]

extract_utils/main.py

@@ -121,6 +121,7 @@ def process_modules(self, source: Source):
                 self.__args.no_cleanup,
                 self.__args.extract_factory,
                 self.__args.section,
+                self.__args.allow_prohibited_files,
             )
             if not copied:
                 all_copied = False

extract_utils/module.py

@@ -50,6 +50,7 @@
     postprocess_carriersettings_fn_impl,
     postprocess_fn_type,
 )
+from extract_utils.prohibited_files import check_prohibited_file
 from extract_utils.source import DiskSource, Source
 from extract_utils.tools import android_root
 from extract_utils.utils import (
@@ -1031,6 +1032,7 @@ def process_file(
         vendor_path: str,
         is_firmware: bool,
         kang: bool,
+        allow_prohibited_files: bool = False,
     ) -> bool:
         file_path = source.get_file_copy_path(file, vendor_path)
 
@@ -1083,6 +1085,9 @@ def process_file(
             )
             return False
 
+        if not allow_prohibited_files:
+            check_prohibited_file(file.dst, file_path)
+
         if kang:
             self.process_kanged_file(
                 file,
@@ -1108,6 +1113,7 @@ def process_proprietary_files(
         backup_source: Source,
         kang: bool,
         extract_factory: bool,
+        allow_prohibited_files: bool = False,
     ) -> bool:
         all_copied = True
 
@@ -1131,6 +1137,7 @@ def process_proprietary_files(
                     vendor_path,
                     is_firmware,
                     kang,
+                    allow_prohibited_files=allow_prohibited_files,
                 )
 
                 if not copied:
@@ -1155,6 +1162,7 @@ def process(
         no_cleanup: bool,
         extract_factory: bool,
         section: Optional[str],
+        allow_prohibited_files: bool = False,
     ):
         with tempfile.TemporaryDirectory() as backup_dir:
             # Kang is usually combined with section, but allow them separately
@@ -1171,4 +1179,5 @@ def process(
                 backup_source,
                 kang,
                 extract_factory,
+                allow_prohibited_files=allow_prohibited_files,
             )

extract_utils/prohibited_files.py

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: The LineageOS Project
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import Callable, List
+
+from extract_utils.utils import Color, color_print
+
+"""
+Prohibited blob policy
+
+This module blocks extraction of prohibited files including:
+
+  - Megvii / Face++ (face recognition, beautification, etc.)
+  - SenseTime license files (e.g. license.lic)
+
+These are disallowed due to licensing, redistribution restrictions,
+and more importantly DMCA takedown risk.
+
+To extend this policy:
+  - Add fnmatch pattern + checker function pairs to PROHIBITED_CHECKS
+"""
+
+
+def _check_sensetime(data: bytes) -> bool:
+    return any(x in data for x in [b'com.sensetime', b'SenseTime'])
+
+
+def _check_megvii(data: bytes) -> bool:
+    return any(x in data for x in [b'megface', b'megvii', b'MEGVII'])
+
+
+# Maps fnmatch pattern (matched against lowercase basename) to a binary
+# checker function. The file is only read if the filename matches.
+PROHIBITED_CHECKS: List[tuple[str, str, Callable[[bytes], bool]]] = [
+    ('*.lic', 'SenseTime', _check_sensetime),
+    ('libmegface*', 'Megvii/Face++', _check_megvii),
+    ('libmegjpeg*', 'Megvii/Face++', _check_megvii),
+    ('libmegskeleton*', 'Megvii/Face++', _check_megvii),
+    ('libmegvii*', 'Megvii/Face++', _check_megvii),
+    ('libmgbeauty*', 'Megvii/Face++', _check_megvii),
+    ('libmgface*', 'Megvii/Face++', _check_megvii),
+]
+
+
+def check_prohibited_file(dst: str, file_path: str):
+    basename = Path(dst).name.lower()
+
+    for pattern, label, checker in PROHIBITED_CHECKS:
+        if not fnmatch(basename, pattern):
+            continue
+        try:
+            data = open(file_path, 'rb').read(4 * 1024 * 1024)
+        except OSError:
+            continue
+        if not checker(data):
+            continue
+
+        color_print(
+            f'ERROR: Prohibited file detected: {dst}',
+            color=Color.RED,
+        )
+        color_print(
+            f'  Reason: {label} binary signature matched in {Path(dst).name}',
+            color=Color.RED,
+        )
+        print()
+        color_print('Policy violation:', color=Color.RED)
+        print(
+            """The following categories of files are not allowed:
+
+   - Megvii / Face++ related libraries and assets:
+    (e.g. lib*{M,m}eg*.so, lib*{M,m}g*.so, *{M,m}egvii*)
+
+   - SenseTime license artifacts:
+    (e.g. license.lic)
+
+These files are not permitted in LineageOS repositories/builds.
+
+Please look for available shims, or develop one to mitigate these dependencies.
+
+To extract them anyway for a private/local build, re-run with:
+
+extract-files.py --allow-prohibited-files [...]
+"""
+        )
+        sys.exit(1)