From e55c766d85c1aa18f6c33f8d6f6639b71d2acf03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Qui=C3=B1ones?= Date: Wed, 21 Jun 2023 08:31:58 -0300 Subject: [PATCH 1/4] Add new option for remote collections host This will be used in next commit. Having the URL configurable will be useful for testing. --- kolibri_explore_plugin/options.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kolibri_explore_plugin/options.py b/kolibri_explore_plugin/options.py index c5db67e75..5555d9960 100644 --- a/kolibri_explore_plugin/options.py +++ b/kolibri_explore_plugin/options.py @@ -8,6 +8,14 @@ to the package install directory. """, }, + "CONTENT_COLLECTIONS_HOST": { + "type": "string", + "default": "https://endlessm.github.io/endless-key-collections", + "description": """ + Remote location where collections manifests are stored. + Defaults to GitHub pages. + """, + }, "CONTENT_COLLECTIONS_PATH": { "type": "string", "default": "", From ed7bcd67c863147e9dcb3682a4cd6a8958f68440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Qui=C3=B1ones?= Date: Wed, 14 Jun 2023 10:18:09 -0300 Subject: [PATCH 2/4] Collection download: Fetch manifests from Github This way the software update is decoupled from content update. Issue #610 --- kolibri_explore_plugin/collectionviews.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/kolibri_explore_plugin/collectionviews.py b/kolibri_explore_plugin/collectionviews.py index 2c33f896c..b7b787e14 100644 --- a/kolibri_explore_plugin/collectionviews.py +++ b/kolibri_explore_plugin/collectionviews.py @@ -4,6 +4,7 @@ from enum import auto from enum import IntEnum +import requests from django.utils.translation import gettext_lazy as _ from kolibri.core.content.errors import InsufficientStorageSpaceError from kolibri.core.content.models import ChannelMetadata @@ -28,6 +29,10 @@ logger = logging.getLogger(__name__) +COLLECTIONS_HOST = conf.OPTIONS["Explore"]["CONTENT_COLLECTIONS_HOST"] + +COLLECTION_URL_TEMPLATE = COLLECTIONS_HOST + "/{grade}-{name}.json" + COLLECTION_PATHS = os.path.join( os.path.dirname(__file__), "static", "collections" ) @@ -77,6 +82,14 @@ def get_extra_channel_ids(self): all_channel_ids = _get_channel_ids_for_all_content_manifests() return all_channel_ids.difference(self.get_channel_ids()) + def read_from_remote_collection(self, grade, name, validate=False): + self.grade = grade + self.name = name + manifest_url = COLLECTION_URL_TEMPLATE.format(grade=grade, name=name) + response = requests.get(manifest_url) + response.raise_for_status() + self.read_dict(response.json(), validate) + def read_from_static_collection(self, grade, name, validate=False): self.grade = grade self.name = name @@ -603,7 +616,7 @@ def _create_manifest(grade, name): try: # TODO: Validate the manifest files or remove validation # https://phabricator.endlessm.com/T34355 - manifest.read_from_static_collection(grade, name, validate=False) + manifest.read_from_remote_collection(grade, name, validate=False) except ContentManifestParseError as err: logger.error(err) else: From fd2d668ec022d517fc31c4eec082dbb6601a996f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Qui=C3=B1ones?= Date: Wed, 14 Jun 2023 10:52:56 -0300 Subject: [PATCH 3/4] Collection download: Initiate on first API call Previously the manifest files were read from disk so it didn't matter to read them all on module import. But now they are fetched from the Internet. So add a decorator to all API views for initiating the global variables and fetch collections only once on first API call. --- kolibri_explore_plugin/collectionviews.py | 25 ++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/kolibri_explore_plugin/collectionviews.py b/kolibri_explore_plugin/collectionviews.py index b7b787e14..d00522e0d 100644 --- a/kolibri_explore_plugin/collectionviews.py +++ b/kolibri_explore_plugin/collectionviews.py @@ -602,12 +602,15 @@ def _build_remotechannelimport_task(channel_id): _content_manifests = [] _content_manifests_by_grade_name = {} -_collection_download_manager = CollectionDownloadManager() +_collection_download_manager = None -def _read_content_manifests(): +def _initiate(): global _content_manifests global _content_manifests_by_grade_name + global _collection_download_manager + + _collection_download_manager = CollectionDownloadManager() free_space_gb = get_free_space() / 1024**3 @@ -631,7 +634,15 @@ def _create_manifest(grade, name): _create_manifest(grade, name) -_read_content_manifests() +def ensure_initiated(api_function): + """Decorator to initiate only once in the first API call.""" + + def wrapper(*args, **kwargs): + if _collection_download_manager is None: + _initiate() + return api_function(*args, **kwargs) + + return wrapper def _save_state_in_request_session(request): @@ -670,6 +681,7 @@ def _get_channel_metadata(channel_id): return ChannelMetadata.objects.get(id=channel_id) +@ensure_initiated @api_view(["GET"]) def get_collection_info(request): """Return the collection metadata and availability.""" @@ -679,6 +691,7 @@ def get_collection_info(request): return Response({"collectionInfo": collection_info}) +@ensure_initiated @api_view(["GET"]) def get_all_collections_info(request): """Return all the collections metadata and their availability.""" @@ -697,6 +710,7 @@ def get_all_collections_info(request): return Response({"allCollectionsInfo": info}) +@ensure_initiated @api_view(["GET"]) def get_should_resume(request): """Return if there is a saved state that should be resumed.""" @@ -715,6 +729,7 @@ def get_should_resume(request): ) +@ensure_initiated @api_view(["POST"]) def start_download(request): """Start downloading a collection. @@ -756,6 +771,7 @@ def start_download(request): return Response({"status": status}) +@ensure_initiated @api_view(["POST"]) def resume_download(request): """Resume download from a previous session. @@ -784,6 +800,7 @@ def resume_download(request): return Response({"status": status}) +@ensure_initiated @api_view(["POST"]) def update_download(request): """Continue downloading current collection. @@ -806,6 +823,7 @@ def update_download(request): return Response({"status": status}) +@ensure_initiated @api_view(["DELETE"]) def cancel_download(request): """Cancel current download and clear the saved state. @@ -825,6 +843,7 @@ def cancel_download(request): return Response({"status": status}) +@ensure_initiated @api_view(["GET"]) def get_download_status(request): """Return the download status.""" From 2bf1c511c3185db314282bce571d9dd9126def34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Qui=C3=B1ones?= Date: Wed, 21 Jun 2023 08:46:44 -0300 Subject: [PATCH 4/4] Collections download: Cleanup local manifests This is not needed anymore. Remove the setting and the method to parse local manifests. --- .gitignore | 3 --- README.md | 4 ---- kolibri_explore_plugin/collectionviews.py | 21 --------------------- kolibri_explore_plugin/options.py | 8 -------- 4 files changed, 36 deletions(-) diff --git a/.gitignore b/.gitignore index 6af448743..9d6ab75b2 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,3 @@ packages/template-ui/src/overrides # Environment file with secrets /.env - -# Collection files. These are bundled in apps -kolibri_explore_plugin/static/collections/ diff --git a/README.md b/README.md index 8e5a68624..f16504322 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,6 @@ extract them to the `kolibri_explore_plugin/apps` folder. Note that this is not ideal, because you should know where has `pip` installed the plugin. -Download the JSON files from the -[endless-key-collections](https://github.com/endlessm/endless-key-collections/tree/main/json) -repo and place them in `kolibri_explore_plugin/static/collections/`. - Now start Kolibri. You should be able to navigate to `/explore` if `/` doesn't redirect you already. diff --git a/kolibri_explore_plugin/collectionviews.py b/kolibri_explore_plugin/collectionviews.py index d00522e0d..59c67ecee 100644 --- a/kolibri_explore_plugin/collectionviews.py +++ b/kolibri_explore_plugin/collectionviews.py @@ -1,5 +1,4 @@ import logging -import os import time from enum import auto from enum import IntEnum @@ -33,12 +32,6 @@ COLLECTION_URL_TEMPLATE = COLLECTIONS_HOST + "/{grade}-{name}.json" -COLLECTION_PATHS = os.path.join( - os.path.dirname(__file__), "static", "collections" -) -if conf.OPTIONS["Explore"]["CONTENT_COLLECTIONS_PATH"]: - COLLECTION_PATHS = conf.OPTIONS["Explore"]["CONTENT_COLLECTIONS_PATH"] - # FIXME: Rename to PACK_IDS COLLECTION_GRADES = [ "explorer", @@ -90,20 +83,6 @@ def read_from_remote_collection(self, grade, name, validate=False): response.raise_for_status() self.read_dict(response.json(), validate) - def read_from_static_collection(self, grade, name, validate=False): - self.grade = grade - self.name = name - manifest_filename = os.path.join( - COLLECTION_PATHS, f"{grade}-{name}.json" - ) - - if not os.path.exists(manifest_filename): - raise ContentManifestParseError( - f"Collection manifest {manifest_filename} not found" - ) - - super().read(manifest_filename, validate) - def read_dict(self, manifest_data, validate=False): self.metadata = manifest_data.get("metadata") if self.metadata is None: diff --git a/kolibri_explore_plugin/options.py b/kolibri_explore_plugin/options.py index 5555d9960..2461c0ead 100644 --- a/kolibri_explore_plugin/options.py +++ b/kolibri_explore_plugin/options.py @@ -16,14 +16,6 @@ Defaults to GitHub pages. """, }, - "CONTENT_COLLECTIONS_PATH": { - "type": "string", - "default": "", - "description": """ - Location where collections manifests are stored. Defaults - to the static/collections folder. - """, - }, "SHOW_AS_STANDALONE_CHANNEL": { "type": "boolean", "default": False,