diff --git a/Jenkinsfile b/Jenkinsfile index 2c27088f..e4446071 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ pipeline { // URL for the Kolibri wheel to include. // FIXME: It would be nice to cache this somehow. // FIXME: Go back to use an official release once that happens on GitHub for 0.16. - KOLIBRI_WHL_URL = 'https://github.com/endlessm/kolibri-explore-plugin/releases/download/v5.8.0/kolibri-0.16.0.dev0+git.20220928203123-py2.py3-none-any.whl' + KOLIBRI_WHL_URL = 'https://github.com/learningequality/kolibri/releases/download/v0.16.0-alpha12/kolibri-0.16.0a12-py2.py3-none-any.whl' // Both p4a and gradle cache outputs in the home directory. // Point it inside the workspace. diff --git a/Makefile b/Makefile index 91b5cbcd..19b6d761 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,13 @@ src/kolibri: clean sed -i 's/if name.endswith(".py"):/if name.endswith(".py") or name.endswith(".pyc"):/g' src/kolibri/dist/django/db/migrations/loader.py # Apply kolibri patches patch -d src/ -p1 < patches/0001-Add-track-progress-information-to-channelimport.patch + patch -d src/ -p1 < patches/0001-Break-up-DynamicWhiteNoise-code.patch + +src/evil_kolibri: src/kolibri + mkdir -p src/evil_kolibri/utils + touch src/evil_kolibri/__init__.py + touch src/evil_kolibri/utils/__init__.py + cp src/kolibri/utils/kolibri_whitenoise.py src/evil_kolibri/utils/kolibri_whitenoise.py .PHONY: apps-bundle.zip apps-bundle.zip: @@ -157,6 +164,7 @@ dist/version.json: needs-version DIST_DEPS = \ p4a_android_distro \ src/kolibri \ + src/evil_kolibri \ src/apps-bundle \ src/collections \ assets/welcomeScreen \ diff --git a/patches/0001-Break-up-DynamicWhiteNoise-code.patch b/patches/0001-Break-up-DynamicWhiteNoise-code.patch new file mode 100644 index 00000000..70f540cb --- /dev/null +++ b/patches/0001-Break-up-DynamicWhiteNoise-code.patch @@ -0,0 +1,130 @@ +From 585771ca61fd6da449cce31425c2ffab322b0dd8 Mon Sep 17 00:00:00 2001 +From: Dylan McCall +Date: Thu, 4 May 2023 17:52:48 -0700 +Subject: [PATCH] Break up DynamicWhiteNoise code + +Instead of generating everything in the initializer, we will use some +more specific functions. This makes it easier to create a subclass of +DynamicWhiteNoise, for example to support platforms that require special +libraries for file access. +--- + kolibri/utils/kolibri_whitenoise.py | 61 ++++++++++++++++++++++------- + 1 file changed, 46 insertions(+), 15 deletions(-) + +diff --git a/kolibri/utils/kolibri_whitenoise.py b/kolibri/utils/kolibri_whitenoise.py +index 828324cf17..4acf1dab21 100644 +--- a/kolibri/utils/kolibri_whitenoise.py ++++ b/kolibri/utils/kolibri_whitenoise.py +@@ -8,6 +8,7 @@ from wsgiref.headers import Headers + from django.contrib.staticfiles import finders + from django.core.files.storage import FileSystemStorage + from django.utils._os import safe_join ++from django.utils.functional import cached_property + from six.moves.urllib.parse import parse_qs + from six.moves.urllib.parse import urljoin + from whitenoise import WhiteNoise +@@ -206,33 +207,57 @@ class DynamicWhiteNoise(WhiteNoise): + } + kwargs.update(whitenoise_settings) + super(DynamicWhiteNoise, self).__init__(application, **kwargs) +- self.dynamic_finder = FileFinder(dynamic_locations or []) ++ self._dynamic_locations = dynamic_locations ++ self._writable_locations = writable_locations ++ self._app_paths = app_paths ++ ++ if static_prefix is not None and not static_prefix.endswith("/"): ++ raise ValueError("Static prefix must end in '/'") ++ self.static_prefix = static_prefix ++ ++ @cached_property ++ def app_path_check(self): ++ # Generate a regex to check if a path patches one of our app paths ++ return ( ++ re.compile("^({})".format("|".join(self._app_paths))) ++ if self._app_paths ++ else None ++ ) ++ ++ @cached_property ++ def dynamic_finder(self): ++ return FileFinder(self._dynamic_locations or []) ++ ++ @cached_property ++ def dynamic_check(self): + # Generate a regex to check if a path matches one of our dynamic + # location prefixes +- self.dynamic_check = ( ++ return ( + re.compile("^({})".format("|".join(self.dynamic_finder.prefixes))) + if self.dynamic_finder.prefixes + else None + ) +- self.writable_locations = {} +- if dynamic_locations: +- for index in writable_locations: ++ ++ @cached_property ++ def writable_locations(self): ++ result = {} ++ if self._dynamic_locations: ++ for index in self._writable_locations: + try: + prefix, root = self.dynamic_finder.locations[index] +- self.writable_locations[prefix] = root ++ result[prefix] = root + except IndexError: + pass +- self.writable_check = ( ++ return result ++ ++ @cached_property ++ def writable_check(self): ++ # Generate a regex to check if a path matches a writable location ++ return ( + re.compile("^({})".format("|".join(self.writable_locations.keys()))) + if self.writable_locations + else None + ) +- self.app_path_check = ( +- re.compile("^({})".format("|".join(app_paths))) if app_paths else None +- ) +- if static_prefix is not None and not static_prefix.endswith("/"): +- raise ValueError("Static prefix must end in '/'") +- self.static_prefix = static_prefix + + def __call__(self, environ, start_response): + path = decode_path_info(environ.get("PATH_INFO", "")) +@@ -308,13 +333,16 @@ class DynamicWhiteNoise(WhiteNoise): + headers["Access-Control-Allow-Origin"] = "*" + if self.add_headers_function: + self.add_headers_function(headers, path, url) +- return EndRangeStaticFile( ++ return self._create_end_range_static_file( + path, + headers.items(), + stat_cache=stat_cache, + encodings={"gzip": path + ".gz", "br": path + ".br"}, + ) + ++ def _create_end_range_static_file(self, path, headers, **kwargs): ++ return EndRangeStaticFile(path, headers, **kwargs) ++ + def get_streaming_static_file(self, url, remote_baseurl): + """ + Vendor this function from source to substitute in our +@@ -334,8 +362,11 @@ class DynamicWhiteNoise(WhiteNoise): + if self.add_headers_function: + self.add_headers_function(headers, path, url) + headers["Content-Encoding"] = "" +- return StreamingStaticFile( ++ return self._create_streaming_range_static_file( + os.path.join(local_dir, path), + headers, + urljoin(remote_baseurl, url.lstrip("/")), + ) ++ ++ def _create_streaming_range_static_file(self, path, headers, remote_url, **kwargs): ++ return StreamingStaticFile(path, headers, remote_url, **kwargs) +-- +2.40.1 + diff --git a/src/kolibri_android/android_whitenoise.py b/src/kolibri_android/android_whitenoise.py index ae7d4567..3471e7f8 100644 --- a/src/kolibri_android/android_whitenoise.py +++ b/src/kolibri_android/android_whitenoise.py @@ -1,48 +1,91 @@ import logging -import os -import re import stat from urllib.parse import urlparse -from wsgiref.headers import Headers -from django.contrib.staticfiles import finders -from django.utils._os import safe_join +from django.utils.functional import cached_property +from evil_kolibri.utils.kolibri_whitenoise import compressed_file_extensions +from evil_kolibri.utils.kolibri_whitenoise import DynamicWhiteNoise +from evil_kolibri.utils.kolibri_whitenoise import EndRangeStaticFile +from evil_kolibri.utils.kolibri_whitenoise import FileFinder from jnius import autoclass -from kolibri.utils.kolibri_whitenoise import compressed_file_extensions -from kolibri.utils.kolibri_whitenoise import EndRangeStaticFile -from kolibri.utils.kolibri_whitenoise import FileFinder -from kolibri.utils.kolibri_whitenoise import NOT_FOUND -from whitenoise import WhiteNoise from whitenoise.httpstatus_backport import HTTPStatus -from whitenoise.responders import MissingFileError from whitenoise.responders import NOT_ALLOWED_RESPONSE from whitenoise.responders import Response -from whitenoise.string_utils import decode_path_info from .android_utils import document_exists from .android_utils import document_tree_join -from .android_utils import get_activity from .android_utils import is_document_uri from .android_utils import open_file from .android_utils import stat_file +# We import from evil_kolibri so we can monkey-patch DynamicWhiteNoise from +# kolibri without creating a circular dependency. + logger = logging.getLogger(__name__) Uri = autoclass("android.net.Uri") -class AndroidFileFinder(FileFinder): - def __init__(self, locations, context, content_resolver): - super().__init__(locations) - self.context = context - self.content_resolver = content_resolver +class AndroidDynamicWhiteNoise(DynamicWhiteNoise): + @staticmethod + def encode_root(root): + if urlparse(root).scheme: + root = "/" + root + return root + + @staticmethod + def decode_root(root): + decoded = root.lstrip("/") + if urlparse(decoded).scheme: + root = decoded + return root + + @classmethod + def decode_locations(cls, locations): + return [(prefix, cls.decode_root(root)) for prefix, root in locations] + + @cached_property + def dynamic_finder(self): + return AndroidFileFinder(self.decode_locations(self._dynamic_locations or [])) + + def find_and_cache_dynamic_file(self, url, remote_baseurl): + path = self.get_dynamic_path(url) + if path: + file_stat = stat_file(path) + # Only try to do matches for regular files. + if stat.S_ISREG(file_stat.st_mode): + stat_cache = {path: file_stat} + for ext in compressed_file_extensions: + try: + comp_path = "{}.{}".format(path, ext) + stat_cache[comp_path] = stat_file(comp_path) + except (IOError, OSError): + pass + self.add_file_to_dictionary(url, path, stat_cache=stat_cache) + return self.files.get(url) + else: + return super().find_and_cache_dynamic_file(url, remote_baseurl) + + def _create_end_range_static_file(self, path, headers, **kwargs): + logger.info(f"Creating EndRangeStaticFile for path {path}") + return AndroidEndRangeStaticFile(path, headers, **kwargs) + + def _create_streaming_range_static_file(self, path, headers, remote_url, **kwargs): + # TODO: We should probably implement this. + logger.info(f"Creating StreamingStaticFile for path {path}") + raise NotImplementedError() + +class AndroidFileFinder(FileFinder): + @cached_property + def document_roots(self): # Create a set of location roots that are DocumentsProvider URIs # to avoid calling isDocumentUri() repeatedly. - self.document_roots = set() + result = set() for _, root in self.locations: - if is_document_uri(root, self.context): - self.document_roots.add(root) + if is_document_uri(root): + result.add(root) + return result def find_location(self, root, path, prefix=None): """ @@ -50,29 +93,30 @@ def find_location(self, root, path, prefix=None): absolute path (or ``None`` if no match). Vendored from Django to handle being passed a URL path instead of a file path. """ + + logger.info(f"Finding path {path} in root {root}") + + document_uri = self.__get_document_uri(root, path, prefix=prefix) + + if document_uri and document_exists(document_uri): + return document_uri.toString() + else: + return super().find_location(root, path, prefix=prefix) + + def __get_document_uri(self, root, path, prefix=None): + if root not in self.document_roots: + return None + if prefix: prefix = prefix + "/" if not path.startswith(prefix): return None path = path[len(prefix) :] - logger.info("Finding path %s in root %s", path, root) - if root in self.document_roots: - path_uri = document_tree_join(Uri.parse(root), path) - if document_exists(path_uri, self.content_resolver): - return path_uri.toString() - else: - path = safe_join(root, path) - if os.path.exists(path): - return path + return document_tree_join(Uri.parse(root), path) class AndroidEndRangeStaticFile(EndRangeStaticFile): - def __init__(self, path, headers, context, content_resolver, **kwargs): - super().__init__(path, headers, **kwargs) - self.context = context - self.content_resolver = content_resolver - def get_response(self, method, request_headers): if method not in ("GET", "HEAD"): return NOT_ALLOWED_RESPONSE @@ -80,9 +124,7 @@ def get_response(self, method, request_headers): return self.not_modified_response path, headers = self.get_path_and_headers(request_headers) if method != "HEAD": - file_handle = open_file( - path, "rb", context=self.context, content_resolver=self.content_resolver - ) + file_handle = open_file(path, "rb") else: file_handle = None range_header = request_headers.get("HTTP_RANGE") @@ -95,130 +137,3 @@ def get_response(self, method, request_headers): # behaviour is allowed by the spec) pass return Response(HTTPStatus.OK, headers, file_handle) - - -class DynamicWhiteNoise(WhiteNoise): - index_file = "index.html" - - def __init__( - self, application, dynamic_locations=None, static_prefix=None, **kwargs - ): - whitenoise_settings = { - # Use 120 seconds as the default cache time for static assets - "max_age": 120, - # Add a test for any file name that contains a semantic version number - # or a 32 digit number (assumed to be a file hash) - # these files will be cached indefinitely - "immutable_file_test": r"((0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)|[a-f0-9]{32})", - "autorefresh": os.environ.get("KOLIBRI_DEVELOPER_MODE", False), - } - kwargs.update(whitenoise_settings) - super(DynamicWhiteNoise, self).__init__(application, **kwargs) - self.context = get_activity() - self.content_resolver = self.context.getContentResolver() - self.dynamic_finder = AndroidFileFinder( - self.decode_locations(dynamic_locations or []), - self.context, - self.content_resolver, - ) - # Generate a regex to check if a path matches one of our dynamic - # location prefixes - self.dynamic_check = ( - re.compile("^({})".format("|".join(self.dynamic_finder.prefixes))) - if self.dynamic_finder.prefixes - else None - ) - if static_prefix is not None and not static_prefix.endswith("/"): - raise ValueError("Static prefix must end in '/'") - self.static_prefix = static_prefix - - def __call__(self, environ, start_response): - path = decode_path_info(environ.get("PATH_INFO", "")) - if self.autorefresh: - static_file = self.find_file(path) - else: - static_file = self.files.get(path) - if static_file is None: - static_file = self.find_and_cache_dynamic_file(path) - if static_file is None: - return self.application(environ, start_response) - return self.serve(static_file, environ, start_response) - - def find_and_cache_dynamic_file(self, url): - path = self.get_dynamic_path(url) - if path: - file_stat = stat_file(path, self.context, self.content_resolver) - # Only try to do matches for regular files. - if stat.S_ISREG(file_stat.st_mode): - stat_cache = {path: file_stat} - for ext in compressed_file_extensions: - try: - comp_path = "{}.{}".format(path, ext) - stat_cache[comp_path] = stat_file( - comp_path, self.context, self.content_resolver - ) - except (IOError, OSError): - pass - self.add_file_to_dictionary(url, path, stat_cache=stat_cache) - elif ( - path is None - and self.static_prefix is not None - and url.startswith(self.static_prefix) - ): - self.files[url] = NOT_FOUND - return self.files.get(url) - - def get_dynamic_path(self, url): - if self.static_prefix is not None and url.startswith(self.static_prefix): - return finders.find(url[len(self.static_prefix) :]) - if self.dynamic_check is not None and self.dynamic_check.match(url): - return self.dynamic_finder.find(url) - - def candidate_paths_for_url(self, url): - paths = super(DynamicWhiteNoise, self).candidate_paths_for_url(url) - for path in paths: - yield path - path = self.get_dynamic_path(url) - if path: - yield path - - def get_static_file(self, path, url, stat_cache=None): - """ - Vendor this function from source to substitute in our - own StaticFile class that can properly handle ranges. - """ - # Optimization: bail early if file does not exist - if stat_cache is None and not os.path.exists(path): - raise MissingFileError(path) - headers = Headers([]) - self.add_mime_headers(headers, path, url) - self.add_cache_headers(headers, path, url) - if self.allow_all_origins: - headers["Access-Control-Allow-Origin"] = "*" - if self.add_headers_function: - self.add_headers_function(headers, path, url) - return AndroidEndRangeStaticFile( - path, - headers.items(), - self.context, - self.content_resolver, - stat_cache=stat_cache, - encodings={"gzip": path + ".gz", "br": path + ".br"}, - ) - - @staticmethod - def encode_root(root): - if urlparse(root).scheme: - root = "/" + root - return root - - @staticmethod - def decode_root(root): - decoded = root.lstrip("/") - if urlparse(decoded).scheme: - root = decoded - return root - - @classmethod - def decode_locations(cls, locations): - return [(prefix, cls.decode_root(root)) for prefix, root in locations] diff --git a/src/kolibri_android/kolibri_utils.py b/src/kolibri_android/kolibri_utils.py index a40c6570..f5960fbd 100644 --- a/src/kolibri_android/kolibri_utils.py +++ b/src/kolibri_android/kolibri_utils.py @@ -10,7 +10,7 @@ from .android_utils import get_signature_key_issuing_organization from .android_utils import get_timezone_name from .android_utils import get_version_name -from .android_whitenoise import DynamicWhiteNoise +from .android_whitenoise import AndroidDynamicWhiteNoise from .globals import SCRIPT_PATH logger = logging.getLogger(__name__) @@ -114,7 +114,9 @@ def _update_kolibri_content_fallback_dirs(): if endless_key_uris is None: return - content_fallback_dirs = DynamicWhiteNoise.encode_root(endless_key_uris["content"]) + content_fallback_dirs = AndroidDynamicWhiteNoise.encode_root( + endless_key_uris["content"] + ) logger.info("Setting KOLIBRI_CONTENT_FALLBACK_DIRS to %s", content_fallback_dirs) os.environ["KOLIBRI_CONTENT_FALLBACK_DIRS"] = content_fallback_dirs @@ -124,7 +126,7 @@ def _monkeypatch_whitenoise(): from kolibri.utils import kolibri_whitenoise logger.info("Applying DynamicWhiteNoise workarounds") - kolibri_whitenoise.DynamicWhiteNoise = DynamicWhiteNoise + kolibri_whitenoise.DynamicWhiteNoise = AndroidDynamicWhiteNoise def _kolibri_initialize(**kwargs):