From 878907c368137ab4f2ef31fe1097a1804fb2ff83 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Wed, 8 Apr 2026 13:37:26 -0400 Subject: [PATCH 01/33] adding aggregation generation method --- vector_search/utils.py | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/vector_search/utils.py b/vector_search/utils.py index d5a50e05f0..784edf20ec 100644 --- a/vector_search/utils.py +++ b/vector_search/utils.py @@ -1,3 +1,4 @@ +import asyncio import gc import logging import uuid @@ -981,6 +982,65 @@ def document_exists(document, collection_name=RESOURCES_COLLECTION_NAME): return count_result.count > 0 +async def async_qdrant_aggregations( + aggregation_keys: list, + facet_filter, + collection_name: str = RESOURCES_COLLECTION_NAME, +) -> dict: + """ + Compute facet aggregations from Qdrant for each requested field. + Issues one concurrent facet query per aggregation key and returns results + in the same shape used by the OpenSearch aggregation API: + ``{"delivery": [{"key": "online", "doc_count": 24}, ...], ...}`` + Args: + aggregation_keys: list of aggregation parameter names. + Must be valid keys in the collection's param map + (e.g. ``QDRANT_RESOURCE_PARAM_MAP``). + search_filter: Qdrant ``models.Filter`` to apply to every facet query, + or ``None`` to aggregate over the whole collection. + collection_name: name of the Qdrant collection to query. + Returns: + dict mapping each requested aggregation name to a list of + ``{"key": str, "doc_count": int}`` dicts sorted by + ``doc_count`` descending. + """ + if not aggregation_keys: + return {} + + collection_param_map = { + RESOURCES_COLLECTION_NAME: QDRANT_RESOURCE_PARAM_MAP, + TOPICS_COLLECTION_NAME: QDRANT_TOPICS_PARAM_MAP, + CONTENT_FILES_COLLECTION_NAME: QDRANT_CONTENT_FILE_PARAM_MAP, + } + param_map = collection_param_map.get(collection_name, QDRANT_RESOURCE_PARAM_MAP) + client = async_qdrant_client() + + async def _get_facet(agg_key: str): + qdrant_field = param_map.get(agg_key) + if not qdrant_field: + return agg_key, [] + result = await client.facet( + collection_name=collection_name, + key=qdrant_field, + facet_filter=facet_filter, + limit=100, + ) + hits = [ + { + "key": str(hit.value).lower() + if isinstance(hit.value, bool) + else str(hit.value), + "doc_count": hit.count, + } + for hit in result.hits + ] + hits.sort(key=lambda x: x["doc_count"], reverse=True) + return agg_key, hits + + results = await asyncio.gather(*[_get_facet(key) for key in aggregation_keys]) + return dict(results) + + def qdrant_query_conditions(params, collection_name=RESOURCES_COLLECTION_NAME): """ Return a list of Qdrant FieldCondition objects based on params From d011419233dd3ace5120b0b824f222b14376435e Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 9 Apr 2026 10:56:32 -0400 Subject: [PATCH 02/33] adding aggregations to response --- vector_search/serializers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index c4c2e7a56a..868ab1341d 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -20,6 +20,7 @@ SearchResponseMetadata, SearchResponseSerializer, ) +from vector_search.constants import QDRANT_RESOURCE_PARAM_MAP class LearningResourcesVectorSearchRequestSerializer(serializers.Serializer): @@ -35,6 +36,17 @@ class LearningResourcesVectorSearchRequestSerializer(serializers.Serializer): limit = serializers.IntegerField( required=False, help_text="Number of results to return per page" ) + aggregation_choices = [ + (key, key.replace("_", " ").title()) for key in QDRANT_RESOURCE_PARAM_MAP + ] + aggregations = serializers.ListField( + required=False, + child=serializers.ChoiceField(choices=aggregation_choices), + help_text=( + f"aggregations for facet counts \ + \n\n{build_choice_description_list(aggregation_choices)}" + ), + ) readable_id = serializers.CharField( required=False, help_text="The readable id of the resource" ) @@ -179,9 +191,9 @@ def get_results(self, instance): def get_count(self, instance) -> int: return instance.get("total", {}).get("value") - def get_metadata(self, _) -> SearchResponseMetadata: + def get_metadata(self, instance) -> SearchResponseMetadata: return { - "aggregations": [], + "aggregations": instance.get("aggregations", []), "suggest": [], } From 8e9aa448542cb7f82e830fbd032575a3a8b3c317 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 9 Apr 2026 10:58:13 -0400 Subject: [PATCH 03/33] adding some optimizations and aggregations to response --- vector_search/constants.py | 6 ++++++ vector_search/views.py | 41 ++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/vector_search/constants.py b/vector_search/constants.py index 0adc4f31a4..80f4da3020 100644 --- a/vector_search/constants.py +++ b/vector_search/constants.py @@ -45,6 +45,7 @@ "title": "title", "url": "url", "resource_type_group": "resource_type_group", + "resource_category": "resource_category", } @@ -71,6 +72,7 @@ "url": models.PayloadSchemaType.KEYWORD, "title": models.PayloadSchemaType.KEYWORD, "resource_type_group": models.PayloadSchemaType.KEYWORD, + "resource_category": models.PayloadSchemaType.KEYWORD, } """ @@ -92,3 +94,7 @@ QDRANT_TOPIC_INDEXES = { "name": models.PayloadSchemaType.KEYWORD, } + + +CONTENT_FILES_RETRIEVE_PAYLOAD = ["key", "run_readable_id"] +RESOURCES_RETRIEVE_PAYLOAD = ["readable_id"] diff --git a/vector_search/views.py b/vector_search/views.py index 4466edd129..7af24a94ef 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -17,6 +17,12 @@ from authentication.decorators import blocked_ip_exempt from learning_resources.constants import GROUP_CONTENT_FILE_CONTENT_VIEWERS from main.utils import cache_page_for_anonymous_users +from vector_search.constants import ( + CONTENT_FILES_COLLECTION_NAME, + CONTENT_FILES_RETRIEVE_PAYLOAD, + RESOURCES_COLLECTION_NAME, + RESOURCES_RETRIEVE_PAYLOAD, +) from vector_search.serializers import ( ContentFileVectorSearchRequestSerializer, ContentFileVectorSearchResponseSerializer, @@ -24,11 +30,10 @@ LearningResourcesVectorSearchResponseSerializer, ) from vector_search.utils import ( - CONTENT_FILES_COLLECTION_NAME, - RESOURCES_COLLECTION_NAME, _content_file_vector_hits, _merge_dicts, _resource_vector_hits, + async_qdrant_aggregations, async_qdrant_client, dense_encoder, qdrant_query_conditions, @@ -113,8 +118,19 @@ async def async_vector_search( # noqa: PLR0913 "collection_name": search_collection, "query_filter": search_filter, "with_vectors": False, - "with_payload": True, - "search_params": models.SearchParams(indexed_only=True, exact=False), + "with_payload": RESOURCES_RETRIEVE_PAYLOAD + if search_collection == RESOURCES_COLLECTION_NAME + else CONTENT_FILES_RETRIEVE_PAYLOAD, + "search_params": models.SearchParams( + quantization=models.QuantizationSearchParams( + ignore=False, + rescore=True, + oversampling=1, + ), + hnsw_ef=64, + indexed_only=True, + exact=False, + ), "limit": limit, } @@ -185,15 +201,24 @@ async def async_vector_search( # noqa: PLR0913 else: hits = await sync_to_async(_content_file_vector_hits)(search_result) - count_result = await client.count( - collection_name=search_collection, - count_filter=search_filter, - exact=False, + aggregation_keys = params.get("aggregations") or [] + count_result, aggregations = await asyncio.gather( + client.count( + collection_name=search_collection, + count_filter=search_filter, + exact=False, + ), + async_qdrant_aggregations( + aggregation_keys, + search_filter, + collection_name=search_collection, + ), ) return { "hits": hits, "total": {"value": count_result.count}, + "aggregations": aggregations, } def handle_exception(self, exc): From f36d1cbc32a5cd73a460450d82febac63b2b3f45 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 9 Apr 2026 10:58:56 -0400 Subject: [PATCH 04/33] regen spec --- frontends/api/src/generated/v0/api.ts | 43 ++++++++++++++++++++++ openapi/specs/v0.yaml | 52 +++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/frontends/api/src/generated/v0/api.ts b/frontends/api/src/generated/v0/api.ts index 368c911b52..b03139c29f 100644 --- a/frontends/api/src/generated/v0/api.ts +++ b/frontends/api/src/generated/v0/api.ts @@ -11991,6 +11991,7 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( /** * Vector Search for learning resources * @summary Vector Search + * @param {Array} [aggregations] aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category * @param {boolean | null} [certification] True if the learning resource offers a certificate * @param {Array} [certification_type] The type of certificate * `micromasters` - MicroMasters Credential * `professional` - Professional Certificate * `completion` - Certificate of Completion * `none` - No Certificate * @param {Array} [course_feature] The course feature. Possible options are at api/v1/course_features/ @@ -12016,6 +12017,7 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( * @throws {RequiredError} */ vectorLearningResourcesSearchRetrieve: async ( + aggregations?: Array, certification?: boolean | null, certification_type?: Array, course_feature?: Array, @@ -12055,6 +12057,10 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( const localVarHeaderParameter = {} as any const localVarQueryParameter = {} as any + if (aggregations) { + localVarQueryParameter["aggregations"] = aggregations + } + if (certification !== undefined) { localVarQueryParameter["certification"] = certification } @@ -12169,6 +12175,7 @@ export const VectorLearningResourcesSearchApiFp = function ( /** * Vector Search for learning resources * @summary Vector Search + * @param {Array} [aggregations] aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category * @param {boolean | null} [certification] True if the learning resource offers a certificate * @param {Array} [certification_type] The type of certificate * `micromasters` - MicroMasters Credential * `professional` - Professional Certificate * `completion` - Certificate of Completion * `none` - No Certificate * @param {Array} [course_feature] The course feature. Possible options are at api/v1/course_features/ @@ -12194,6 +12201,7 @@ export const VectorLearningResourcesSearchApiFp = function ( * @throws {RequiredError} */ async vectorLearningResourcesSearchRetrieve( + aggregations?: Array, certification?: boolean | null, certification_type?: Array, course_feature?: Array, @@ -12224,6 +12232,7 @@ export const VectorLearningResourcesSearchApiFp = function ( > { const localVarAxiosArgs = await localVarAxiosParamCreator.vectorLearningResourcesSearchRetrieve( + aggregations, certification, certification_type, course_feature, @@ -12287,6 +12296,7 @@ export const VectorLearningResourcesSearchApiFactory = function ( ): AxiosPromise { return localVarFp .vectorLearningResourcesSearchRetrieve( + requestParameters.aggregations, requestParameters.certification, requestParameters.certification_type, requestParameters.course_feature, @@ -12321,6 +12331,13 @@ export const VectorLearningResourcesSearchApiFactory = function ( * @interface VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieveRequest */ export interface VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieveRequest { + /** + * aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category + * @type {Array<'readable_id' | 'resource_type' | 'certification' | 'certification_type' | 'professional' | 'free' | 'course_feature' | 'topic' | 'ocw_topic' | 'level' | 'department' | 'platform' | 'offered_by' | 'delivery' | 'title' | 'url' | 'resource_type_group' | 'resource_category'>} + * @memberof VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieve + */ + readonly aggregations?: Array + /** * True if the learning resource offers a certificate * @type {boolean} @@ -12490,6 +12507,7 @@ export class VectorLearningResourcesSearchApi extends BaseAPI { ) { return VectorLearningResourcesSearchApiFp(this.configuration) .vectorLearningResourcesSearchRetrieve( + requestParameters.aggregations, requestParameters.certification, requestParameters.certification_type, requestParameters.course_feature, @@ -12517,6 +12535,31 @@ export class VectorLearningResourcesSearchApi extends BaseAPI { } } +/** + * @export + */ +export const VectorLearningResourcesSearchRetrieveAggregationsEnum = { + ReadableId: "readable_id", + ResourceType: "resource_type", + Certification: "certification", + CertificationType: "certification_type", + Professional: "professional", + Free: "free", + CourseFeature: "course_feature", + Topic: "topic", + OcwTopic: "ocw_topic", + Level: "level", + Department: "department", + Platform: "platform", + OfferedBy: "offered_by", + Delivery: "delivery", + Title: "title", + Url: "url", + ResourceTypeGroup: "resource_type_group", + ResourceCategory: "resource_category", +} as const +export type VectorLearningResourcesSearchRetrieveAggregationsEnum = + (typeof VectorLearningResourcesSearchRetrieveAggregationsEnum)[keyof typeof VectorLearningResourcesSearchRetrieveAggregationsEnum] /** * @export */ diff --git a/openapi/specs/v0.yaml b/openapi/specs/v0.yaml index 4bb97c19c2..1503316f7a 100644 --- a/openapi/specs/v0.yaml +++ b/openapi/specs/v0.yaml @@ -961,6 +961,58 @@ paths: description: Vector Search for learning resources summary: Vector Search parameters: + - in: query + name: aggregations + schema: + type: array + items: + enum: + - readable_id + - resource_type + - certification + - certification_type + - professional + - free + - course_feature + - topic + - ocw_topic + - level + - department + - platform + - offered_by + - delivery + - title + - url + - resource_type_group + - resource_category + type: string + description: |- + * `readable_id` - Readable Id + * `resource_type` - Resource Type + * `certification` - Certification + * `certification_type` - Certification Type + * `professional` - Professional + * `free` - Free + * `course_feature` - Course Feature + * `topic` - Topic + * `ocw_topic` - Ocw Topic + * `level` - Level + * `department` - Department + * `platform` - Platform + * `offered_by` - Offered By + * `delivery` - Delivery + * `title` - Title + * `url` - Url + * `resource_type_group` - Resource Type Group + * `resource_category` - Resource Category + description: "aggregations for facet counts \n\n* `readable_id`\ + \ - Readable Id\n* `resource_type` - Resource Type\n* `certification` -\ + \ Certification\n* `certification_type` - Certification Type\n* `professional`\ + \ - Professional\n* `free` - Free\n* `course_feature` - Course Feature\n\ + * `topic` - Topic\n* `ocw_topic` - Ocw Topic\n* `level` - Level\n* `department`\ + \ - Department\n* `platform` - Platform\n* `offered_by` - Offered By\n*\ + \ `delivery` - Delivery\n* `title` - Title\n* `url` - Url\n* `resource_type_group`\ + \ - Resource Type Group\n* `resource_category` - Resource Category" - in: query name: certification schema: From ab34a23404482932ef80ddab0fb4d1393b96b1a9 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 9 Apr 2026 14:29:13 -0400 Subject: [PATCH 05/33] add published back to learning resources serializer --- vector_search/constants.py | 1 + vector_search/serializers.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/vector_search/constants.py b/vector_search/constants.py index 80f4da3020..5153d4b24b 100644 --- a/vector_search/constants.py +++ b/vector_search/constants.py @@ -46,6 +46,7 @@ "url": "url", "resource_type_group": "resource_type_group", "resource_category": "resource_category", + "published": "published", } diff --git a/vector_search/serializers.py b/vector_search/serializers.py index 868ab1341d..e6b87920b4 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -47,6 +47,11 @@ class LearningResourcesVectorSearchRequestSerializer(serializers.Serializer): \n\n{build_choice_description_list(aggregation_choices)}" ), ) + published = serializers.BooleanField( + required=False, + default=True, + help_text="If the resource is published. We default to True unless passed in", + ) readable_id = serializers.CharField( required=False, help_text="The readable id of the resource" ) From 22c8d439af5c9222ce2c67d20752e2ceec12298a Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 9 Apr 2026 15:26:17 -0400 Subject: [PATCH 06/33] spec update --- frontends/api/src/generated/v0/api.ts | 27 +++++++++++++++++++++++---- openapi/specs/v0.yaml | 12 +++++++++++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/frontends/api/src/generated/v0/api.ts b/frontends/api/src/generated/v0/api.ts index b03139c29f..84156d2899 100644 --- a/frontends/api/src/generated/v0/api.ts +++ b/frontends/api/src/generated/v0/api.ts @@ -11991,7 +11991,7 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( /** * Vector Search for learning resources * @summary Vector Search - * @param {Array} [aggregations] aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category + * @param {Array} [aggregations] aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category * `published` - Published * @param {boolean | null} [certification] True if the learning resource offers a certificate * @param {Array} [certification_type] The type of certificate * `micromasters` - MicroMasters Credential * `professional` - Professional Certificate * `completion` - Certificate of Completion * `none` - No Certificate * @param {Array} [course_feature] The course feature. Possible options are at api/v1/course_features/ @@ -12006,6 +12006,7 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube * `canvas` - Canvas * `climate` - MIT Climate * `ovs` - ODL Video Service * @param {boolean | null} [professional] + * @param {boolean} [published] If the resource is published. We default to True unless passed in * @param {string} [q] The search text * @param {string} [readable_id] The readable id of the resource * @param {Array} [resource_type] The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document @@ -12032,6 +12033,7 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( offset?: number, platform?: Array, professional?: boolean | null, + published?: boolean, q?: string, readable_id?: string, resource_type?: Array, @@ -12117,6 +12119,10 @@ export const VectorLearningResourcesSearchApiAxiosParamCreator = function ( localVarQueryParameter["professional"] = professional } + if (published !== undefined) { + localVarQueryParameter["published"] = published + } + if (q !== undefined) { localVarQueryParameter["q"] = q } @@ -12175,7 +12181,7 @@ export const VectorLearningResourcesSearchApiFp = function ( /** * Vector Search for learning resources * @summary Vector Search - * @param {Array} [aggregations] aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category + * @param {Array} [aggregations] aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category * `published` - Published * @param {boolean | null} [certification] True if the learning resource offers a certificate * @param {Array} [certification_type] The type of certificate * `micromasters` - MicroMasters Credential * `professional` - Professional Certificate * `completion` - Certificate of Completion * `none` - No Certificate * @param {Array} [course_feature] The course feature. Possible options are at api/v1/course_features/ @@ -12190,6 +12196,7 @@ export const VectorLearningResourcesSearchApiFp = function ( * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube * `canvas` - Canvas * `climate` - MIT Climate * `ovs` - ODL Video Service * @param {boolean | null} [professional] + * @param {boolean} [published] If the resource is published. We default to True unless passed in * @param {string} [q] The search text * @param {string} [readable_id] The readable id of the resource * @param {Array} [resource_type] The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document @@ -12216,6 +12223,7 @@ export const VectorLearningResourcesSearchApiFp = function ( offset?: number, platform?: Array, professional?: boolean | null, + published?: boolean, q?: string, readable_id?: string, resource_type?: Array, @@ -12247,6 +12255,7 @@ export const VectorLearningResourcesSearchApiFp = function ( offset, platform, professional, + published, q, readable_id, resource_type, @@ -12311,6 +12320,7 @@ export const VectorLearningResourcesSearchApiFactory = function ( requestParameters.offset, requestParameters.platform, requestParameters.professional, + requestParameters.published, requestParameters.q, requestParameters.readable_id, requestParameters.resource_type, @@ -12332,8 +12342,8 @@ export const VectorLearningResourcesSearchApiFactory = function ( */ export interface VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieveRequest { /** - * aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category - * @type {Array<'readable_id' | 'resource_type' | 'certification' | 'certification_type' | 'professional' | 'free' | 'course_feature' | 'topic' | 'ocw_topic' | 'level' | 'department' | 'platform' | 'offered_by' | 'delivery' | 'title' | 'url' | 'resource_type_group' | 'resource_category'>} + * aggregations for facet counts * `readable_id` - Readable Id * `resource_type` - Resource Type * `certification` - Certification * `certification_type` - Certification Type * `professional` - Professional * `free` - Free * `course_feature` - Course Feature * `topic` - Topic * `ocw_topic` - Ocw Topic * `level` - Level * `department` - Department * `platform` - Platform * `offered_by` - Offered By * `delivery` - Delivery * `title` - Title * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category * `published` - Published + * @type {Array<'readable_id' | 'resource_type' | 'certification' | 'certification_type' | 'professional' | 'free' | 'course_feature' | 'topic' | 'ocw_topic' | 'level' | 'department' | 'platform' | 'offered_by' | 'delivery' | 'title' | 'url' | 'resource_type_group' | 'resource_category' | 'published'>} * @memberof VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieve */ readonly aggregations?: Array @@ -12436,6 +12446,13 @@ export interface VectorLearningResourcesSearchApiVectorLearningResourcesSearchRe */ readonly professional?: boolean | null + /** + * If the resource is published. We default to True unless passed in + * @type {boolean} + * @memberof VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieve + */ + readonly published?: boolean + /** * The search text * @type {string} @@ -12522,6 +12539,7 @@ export class VectorLearningResourcesSearchApi extends BaseAPI { requestParameters.offset, requestParameters.platform, requestParameters.professional, + requestParameters.published, requestParameters.q, requestParameters.readable_id, requestParameters.resource_type, @@ -12557,6 +12575,7 @@ export const VectorLearningResourcesSearchRetrieveAggregationsEnum = { Url: "url", ResourceTypeGroup: "resource_type_group", ResourceCategory: "resource_category", + Published: "published", } as const export type VectorLearningResourcesSearchRetrieveAggregationsEnum = (typeof VectorLearningResourcesSearchRetrieveAggregationsEnum)[keyof typeof VectorLearningResourcesSearchRetrieveAggregationsEnum] diff --git a/openapi/specs/v0.yaml b/openapi/specs/v0.yaml index 1503316f7a..3a686ef55d 100644 --- a/openapi/specs/v0.yaml +++ b/openapi/specs/v0.yaml @@ -985,6 +985,7 @@ paths: - url - resource_type_group - resource_category + - published type: string description: |- * `readable_id` - Readable Id @@ -1005,6 +1006,7 @@ paths: * `url` - Url * `resource_type_group` - Resource Type Group * `resource_category` - Resource Category + * `published` - Published description: "aggregations for facet counts \n\n* `readable_id`\ \ - Readable Id\n* `resource_type` - Resource Type\n* `certification` -\ \ Certification\n* `certification_type` - Certification Type\n* `professional`\ @@ -1012,7 +1014,8 @@ paths: * `topic` - Topic\n* `ocw_topic` - Ocw Topic\n* `level` - Level\n* `department`\ \ - Department\n* `platform` - Platform\n* `offered_by` - Offered By\n*\ \ `delivery` - Delivery\n* `title` - Title\n* `url` - Url\n* `resource_type_group`\ - \ - Resource Type Group\n* `resource_category` - Resource Category" + \ - Resource Type Group\n* `resource_category` - Resource Category\n* `published`\ + \ - Published" - in: query name: certification schema: @@ -1307,6 +1310,13 @@ paths: schema: type: boolean nullable: true + - in: query + name: published + schema: + type: boolean + default: true + description: If the resource is published. We default to True unless passed + in - in: query name: q schema: From 518c5a14401247702acecc3f7eda5e10143be437 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 9 Apr 2026 15:27:08 -0400 Subject: [PATCH 07/33] show facets on frontend --- .../main/src/page-components/SearchDisplay/SearchDisplay.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx index 7aba279191..b9de34bab1 100644 --- a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -526,6 +526,7 @@ const searchModeDropdownOptions = Object.entries( const toVectorSearchParams = ( params: ReturnType, ): VectorSearchRequest => ({ + aggregations: params.aggregations as VectorSearchRequest["aggregations"], certification: params.certification, certification_type: params.certification_type as VectorSearchRequest["certification_type"], @@ -973,9 +974,7 @@ const SearchDisplay: React.FC = ({ * the count when data is loaded even if count is same as previous * count. */} - {isFetching || isLoading || isVectorSearch - ? "" - : `${data?.count} results`} + {isFetching || isLoading ? "" : `${data?.count} results`} From 73a080ad26d814ab1cb65df6905efeeb4d00e268 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Fri, 10 Apr 2026 15:19:04 -0400 Subject: [PATCH 08/33] fixing aggregation counts --- vector_search/utils.py | 14 +++++++++++--- vector_search/views.py | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/vector_search/utils.py b/vector_search/utils.py index 784edf20ec..1747747f44 100644 --- a/vector_search/utils.py +++ b/vector_search/utils.py @@ -984,7 +984,7 @@ def document_exists(document, collection_name=RESOURCES_COLLECTION_NAME): async def async_qdrant_aggregations( aggregation_keys: list, - facet_filter, + params: dict, collection_name: str = RESOURCES_COLLECTION_NAME, ) -> dict: """ @@ -996,8 +996,8 @@ async def async_qdrant_aggregations( aggregation_keys: list of aggregation parameter names. Must be valid keys in the collection's param map (e.g. ``QDRANT_RESOURCE_PARAM_MAP``). - search_filter: Qdrant ``models.Filter`` to apply to every facet query, - or ``None`` to aggregate over the whole collection. + params: dict of all search parameters, which are used to construct + a Qdrant ``models.Filter`` for each facet query. collection_name: name of the Qdrant collection to query. Returns: dict mapping each requested aggregation name to a list of @@ -1019,6 +1019,14 @@ async def _get_facet(agg_key: str): qdrant_field = param_map.get(agg_key) if not qdrant_field: return agg_key, [] + + filtered_params = { + k: v for k, v in params.items() if k.partition("__")[0] != agg_key + } + facet_filter = qdrant_query_conditions( + filtered_params, collection_name=collection_name + ) + result = await client.facet( collection_name=collection_name, key=qdrant_field, diff --git a/vector_search/views.py b/vector_search/views.py index 7af24a94ef..b82b5a0df3 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -210,7 +210,7 @@ async def async_vector_search( # noqa: PLR0913 ), async_qdrant_aggregations( aggregation_keys, - search_filter, + params, collection_name=search_collection, ), ) From f3168d65e9aee9f7609018a143f3d86eb942aad2 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Fri, 10 Apr 2026 16:12:57 -0400 Subject: [PATCH 09/33] fix test --- .../main/src/page-components/SearchDisplay/SearchDisplay.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx index b9de34bab1..1178b19906 100644 --- a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -625,6 +625,7 @@ const SearchDisplay: React.FC = ({ const isVectorSearch = wantsVectorSearch && user?.is_learning_path_editor const { data, isLoading, isFetching } = useQuery({ + // @ts-expect-error Typescript has trouble unifying the different query key types ...(isVectorSearch ? learningResourceQueries.vectorSearch(toVectorSearchParams(allParams)) : learningResourceQueries.search(allParams as LRSearchRequest)), From 937ddf7e1c743002fc0e3c3b7d3410be2acb3e7d Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Fri, 10 Apr 2026 16:22:59 -0400 Subject: [PATCH 10/33] fix typechecks --- .../page-components/SearchDisplay/SearchDisplay.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx index 1178b19906..51d03537e5 100644 --- a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -624,11 +624,13 @@ const SearchDisplay: React.FC = ({ const wantsVectorSearch = searchParams.get("vector_search") === "true" const isVectorSearch = wantsVectorSearch && user?.is_learning_path_editor + const queryOptions = isVectorSearch + ? learningResourceQueries.vectorSearch(toVectorSearchParams(allParams)) + : learningResourceQueries.search(allParams as LRSearchRequest) + + // @ts-expect-error Typescript has trouble unifying the different query key types const { data, isLoading, isFetching } = useQuery({ - // @ts-expect-error Typescript has trouble unifying the different query key types - ...(isVectorSearch - ? learningResourceQueries.vectorSearch(toVectorSearchParams(allParams)) - : learningResourceQueries.search(allParams as LRSearchRequest)), + ...queryOptions, enabled: !wantsVectorSearch || !isUserLoading, placeholderData: keepPreviousData, select: ( From 1c1ea407f742f4363445739e68693004431f813c Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Fri, 10 Apr 2026 16:47:33 -0400 Subject: [PATCH 11/33] remove unused test --- .../app-pages/SearchPage/SearchPage.test.tsx | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx b/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx index c0ff7c3d97..f5da0fc92e 100644 --- a/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx +++ b/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx @@ -152,50 +152,6 @@ describe("SearchPage", () => { }, ) - test("Vector Hybrid Search passes correct params and hides count", async () => { - setMockApiResponses({ - search: { - count: 700, - metadata: { - aggregations: { - resource_type_group: [{ key: "course", doc_count: 100 }], - }, - suggestions: [], - }, - results: factories.learningResources.resources({ count: 5 }).results, - }, - }) - - // Authenticate as path editor (admin) - setMockResponse.get(urls.userMe.get(), { - is_learning_path_editor: true, - is_authenticated: true, - }) - - renderWithProviders(, { url: "?vector_search=true&q=test" }) - - await waitFor(() => { - const call = makeRequest.mock.calls.find(([_method, url]) => { - return url.includes(urls.search.vectorResources()) - }) - expect(call).toBeDefined() - }) - - const call = makeRequest.mock.calls.find(([_method, url]) => - url.includes(urls.search.vectorResources()), - ) - invariant(call) - const fullUrl = new URL(call[1], "http://mit.edu") - const apiSearchParams = fullUrl.searchParams - - expect(apiSearchParams.get("hybrid_search")).toBe("true") - expect(apiSearchParams.get("q")).toBe("test") - - // Ensure count is hidden - const hideCountText = screen.queryByText("700 results") - expect(hideCountText).toBeNull() - }) - test("Toggling facets", async () => { setMockApiResponses({ search: { From 11dfdc8d3e94759b19f7d1ff1366bcd4698a1977 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Mon, 13 Apr 2026 15:18:46 -0400 Subject: [PATCH 12/33] adding tests for aggregations --- vector_search/utils_test.py | 246 ++++++++++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) diff --git a/vector_search/utils_test.py b/vector_search/utils_test.py index 8917834c5b..c9d2458a44 100644 --- a/vector_search/utils_test.py +++ b/vector_search/utils_test.py @@ -1,3 +1,4 @@ +import asyncio import random from decimal import Decimal from unittest.mock import MagicMock @@ -44,6 +45,7 @@ _get_text_splitter, _is_markdown_content, _resource_vector_hits, + async_qdrant_aggregations, create_qdrant_collections, embed_learning_resources, embed_topics, @@ -1519,3 +1521,247 @@ def test_resource_vector_hits_preserves_qdrant_score_order(): expected_readable_ids = [r.readable_id for r in shuffled] actual_readable_ids = [r["readable_id"] for r in result] assert actual_readable_ids == expected_readable_ids + + +def _make_facet_hit(count=0, value="test"): + """Build a minimal mock that looks like a Qdrant FacetHit.""" + hit = MagicMock() + hit.value = value + hit.count = count + return hit + + +def _make_facet_response(hits): + """Build a minimal mock that looks like a Qdrant FacetResponse.""" + resp = MagicMock() + resp.hits = hits + return resp + + +def test_async_qdrant_aggregations_empty_keys(mocker): + """Should return {} immediately without calling Qdrant when aggregation_keys is empty.""" + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + result = asyncio.run(async_qdrant_aggregations([], {})) + assert result == {} + mock_client.facet.assert_not_called() + + +def test_async_qdrant_aggregations_unknown_key(mocker): + """An aggregation key not present in the param map should return an empty list.""" + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + result = asyncio.run( + async_qdrant_aggregations( + ["nonexistent_field"], + {}, + collection_name=RESOURCES_COLLECTION_NAME, + ) + ) + assert result == {"nonexistent_field": []} + mock_client.facet.assert_not_called() + + +def test_async_qdrant_aggregations_single_key(mocker): + """A valid single aggregation key should query Qdrant and return correctly shaped data.""" + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + + mock_client.facet.return_value = _make_facet_response( + [ + _make_facet_hit(42, value="course"), + _make_facet_hit(7, value="podcast"), + ] + ) + + result = asyncio.run( + async_qdrant_aggregations( + ["resource_type"], + {}, + collection_name=RESOURCES_COLLECTION_NAME, + ) + ) + + assert "resource_type" in result + hits = result["resource_type"] + # Should be sorted descending by doc_count + assert hits[0] == {"key": "course", "doc_count": 42} + assert hits[1] == {"key": "podcast", "doc_count": 7} + + mock_client.facet.assert_awaited_once() + call_kwargs = mock_client.facet.call_args.kwargs + assert call_kwargs["collection_name"] == RESOURCES_COLLECTION_NAME + assert call_kwargs["key"] == QDRANT_RESOURCE_PARAM_MAP["resource_type"] + assert call_kwargs["limit"] == 100 + + +def test_async_qdrant_aggregations_multiple_keys(mocker): + """Multiple valid keys should each issue a concurrent Qdrant facet call.""" + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + + # Return different data depending on the 'key' kwarg + def _facet_side_effect(**kwargs): + if kwargs["key"] == QDRANT_RESOURCE_PARAM_MAP["resource_type"]: + return _make_facet_response([_make_facet_hit(10, value="course")]) + if kwargs["key"] == QDRANT_RESOURCE_PARAM_MAP["platform"]: + return _make_facet_response( + [_make_facet_hit(30, value="ocw"), _make_facet_hit(20, value="edx")] + ) + return _make_facet_response([]) + + mock_client.facet.side_effect = _facet_side_effect + + result = asyncio.run( + async_qdrant_aggregations( + ["resource_type", "platform"], + {}, + collection_name=RESOURCES_COLLECTION_NAME, + ) + ) + + assert set(result.keys()) == {"resource_type", "platform"} + assert result["resource_type"] == [{"key": "course", "doc_count": 10}] + # Descending sort + assert result["platform"][0] == {"key": "ocw", "doc_count": 30} + assert result["platform"][1] == {"key": "edx", "doc_count": 20} + assert mock_client.facet.await_count == 2 + + +def test_async_qdrant_aggregations_excludes_own_param_from_filter(mocker): + """ + When building the per-facet filter, the aggregation key's own param + must be excluded so that all values for that facet are counted. + """ + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + mock_client.facet.return_value = _make_facet_response([]) + + params = { + "resource_type": ["course"], + "platform": ["ocw"], + } + + asyncio.run( + async_qdrant_aggregations( + ["resource_type"], + params, + collection_name=RESOURCES_COLLECTION_NAME, + ) + ) + + mock_client.facet.assert_awaited_once() + call_kwargs = mock_client.facet.call_args.kwargs + + # The facet_filter should NOT contain a condition for resource_type + # (it was stripped out so we get all resource_type facet values), + # but it SHOULD still filter by platform. + facet_filter = call_kwargs.get("facet_filter") + # facet_filter is a qdrant models.Filter with must conditions + assert facet_filter is not None + condition_keys = [c.key for c in facet_filter.must if hasattr(c, "key")] + assert QDRANT_RESOURCE_PARAM_MAP["platform"] in condition_keys + assert QDRANT_RESOURCE_PARAM_MAP["resource_type"] not in condition_keys + + +def test_async_qdrant_aggregations_bool_values_lowercased(mocker): + """Boolean hit values must be returned as lowercase strings ('true'/'false').""" + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + + mock_client.facet.return_value = _make_facet_response( + [ + _make_facet_hit(5, value=True), + _make_facet_hit(3, value=False), + ] + ) + + result = asyncio.run( + async_qdrant_aggregations( + ["free"], + {}, + collection_name=RESOURCES_COLLECTION_NAME, + ) + ) + + keys = {hit["key"] for hit in result["free"]} + assert "true" in keys + assert "false" in keys + # Verify no raw booleans slipped through + assert True not in keys + assert False not in keys + + +def test_async_qdrant_aggregations_sorted_by_doc_count_desc(mocker): + """Results must be sorted by doc_count in descending order.""" + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + + mock_client.facet.return_value = _make_facet_response( + [ + _make_facet_hit(5, value="edx"), + _make_facet_hit(100, value="ocw"), + _make_facet_hit(20, value="xpro"), + ] + ) + + result = asyncio.run( + async_qdrant_aggregations( + ["platform"], + {}, + collection_name=RESOURCES_COLLECTION_NAME, + ) + ) + + counts = [hit["doc_count"] for hit in result["platform"]] + assert counts == sorted(counts, reverse=True) + + +def test_async_qdrant_aggregations_uses_content_file_param_map(mocker): + """ + When collection_name is CONTENT_FILES_COLLECTION_NAME the function must + use QDRANT_CONTENT_FILE_PARAM_MAP to resolve the Qdrant field name. + """ + mock_client = mocker.AsyncMock() + mocker.patch( + "vector_search.utils.async_qdrant_client", + return_value=mock_client, + ) + mock_client.facet.return_value = _make_facet_response( + [_make_facet_hit(8, value=".pdf")] + ) + + result = asyncio.run( + async_qdrant_aggregations( + ["file_extension"], + {}, + collection_name=CONTENT_FILES_COLLECTION_NAME, + ) + ) + + assert "file_extension" in result + call_kwargs = mock_client.facet.call_args.kwargs + assert call_kwargs["collection_name"] == CONTENT_FILES_COLLECTION_NAME + # The Qdrant field for 'file_extension' should come from the content-file map + assert call_kwargs["key"] == QDRANT_CONTENT_FILE_PARAM_MAP["file_extension"] From 23d9bb4f4e91b01c404523e350e5570f05569a2a Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Mon, 13 Apr 2026 15:49:41 -0400 Subject: [PATCH 13/33] Update vector_search/serializers.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- vector_search/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index e6b87920b4..bd1b5a5fbe 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -198,7 +198,7 @@ def get_count(self, instance) -> int: def get_metadata(self, instance) -> SearchResponseMetadata: return { - "aggregations": instance.get("aggregations", []), + "aggregations": instance.get("aggregations", {}), "suggest": [], } From 7abd6f90f9d8307dab75078a768c45f77b40b6f8 Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Mon, 13 Apr 2026 15:49:53 -0400 Subject: [PATCH 14/33] Update vector_search/views.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- vector_search/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector_search/views.py b/vector_search/views.py index b82b5a0df3..c8f1a05823 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -218,7 +218,7 @@ async def async_vector_search( # noqa: PLR0913 return { "hits": hits, "total": {"value": count_result.count}, - "aggregations": aggregations, + "aggregations": aggregations or {}, } def handle_exception(self, exc): From 45a3011a95ae4258e6d562852f66b4e37ee784a6 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 11:08:37 -0400 Subject: [PATCH 15/33] fixing 'with_payload' for group by --- vector_search/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vector_search/views.py b/vector_search/views.py index c8f1a05823..fbca35c888 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -167,6 +167,7 @@ async def async_vector_search( # noqa: PLR0913 search_params.pop("search_params", None) search_params["group_by"] = params.get("group_by") search_params["group_size"] = params.get("group_size", 1) + search_params["with_payload"] = True group_result = await client.query_points_groups(**search_params) search_result = [] for group in group_result.groups: From ab4c0f8d456718d3a843a8d4421f293ecda15541 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 11:10:22 -0400 Subject: [PATCH 16/33] switch to safe getter --- vector_search/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector_search/utils.py b/vector_search/utils.py index 1747747f44..8025ebcc95 100644 --- a/vector_search/utils.py +++ b/vector_search/utils.py @@ -872,7 +872,7 @@ def process_batch(docs_batch, summaries_list): def _resource_vector_hits(search_result): - hits = [hit.payload["readable_id"] for hit in search_result] + hits = [hit.payload.get("readable_id") for hit in search_result] """ Always lookup learning resources by readable_id for portability in case we load points from external systems From d310643c59a0181e6aeae4e65a563284e1f09420 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 11:12:26 -0400 Subject: [PATCH 17/33] correct comment about dropping admin params --- .../main/src/page-components/SearchDisplay/SearchDisplay.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx index 51d03537e5..92ca6dd5f1 100644 --- a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -516,8 +516,8 @@ const searchModeDropdownOptions = Object.entries( /** * Extracts only the fields supported by the vector search API from a broader - * search params object, dropping admin-only params (e.g., aggregations, - * content_file_score_weight) that the vector endpoint does not accept. + * search params object, dropping admin-only params (e.g., content_file_score_weight) + * that the vector endpoint does not accept. * * The `as` casts for enum arrays are safe because the v0 and v1 generated * clients define separate (but structurally identical) enum types for the same From ef6c22770ed0232a2e436204afc2d30f883821ac Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 11:14:35 -0400 Subject: [PATCH 18/33] switching collection param map to constant --- vector_search/constants.py | 7 +++++++ vector_search/utils.py | 16 +++------------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/vector_search/constants.py b/vector_search/constants.py index 5153d4b24b..cadd81622a 100644 --- a/vector_search/constants.py +++ b/vector_search/constants.py @@ -99,3 +99,10 @@ CONTENT_FILES_RETRIEVE_PAYLOAD = ["key", "run_readable_id"] RESOURCES_RETRIEVE_PAYLOAD = ["readable_id"] + + +COLLECTION_PARAM_MAP = { + RESOURCES_COLLECTION_NAME: QDRANT_RESOURCE_PARAM_MAP, + TOPICS_COLLECTION_NAME: QDRANT_TOPICS_PARAM_MAP, + CONTENT_FILES_COLLECTION_NAME: QDRANT_CONTENT_FILE_PARAM_MAP, +} diff --git a/vector_search/utils.py b/vector_search/utils.py index 8025ebcc95..0027445da4 100644 --- a/vector_search/utils.py +++ b/vector_search/utils.py @@ -33,13 +33,13 @@ ) from main.utils import checksum_for_content from vector_search.constants import ( + COLLECTION_PARAM_MAP, CONTENT_FILES_COLLECTION_NAME, QDRANT_CONTENT_FILE_INDEXES, QDRANT_CONTENT_FILE_PARAM_MAP, QDRANT_LEARNING_RESOURCE_INDEXES, QDRANT_RESOURCE_PARAM_MAP, QDRANT_TOPIC_INDEXES, - QDRANT_TOPICS_PARAM_MAP, RESOURCES_COLLECTION_NAME, TOPICS_COLLECTION_NAME, ) @@ -1007,12 +1007,7 @@ async def async_qdrant_aggregations( if not aggregation_keys: return {} - collection_param_map = { - RESOURCES_COLLECTION_NAME: QDRANT_RESOURCE_PARAM_MAP, - TOPICS_COLLECTION_NAME: QDRANT_TOPICS_PARAM_MAP, - CONTENT_FILES_COLLECTION_NAME: QDRANT_CONTENT_FILE_PARAM_MAP, - } - param_map = collection_param_map.get(collection_name, QDRANT_RESOURCE_PARAM_MAP) + param_map = COLLECTION_PARAM_MAP.get(collection_name, QDRANT_RESOURCE_PARAM_MAP) client = async_qdrant_client() async def _get_facet(agg_key: str): @@ -1054,12 +1049,7 @@ def qdrant_query_conditions(params, collection_name=RESOURCES_COLLECTION_NAME): Return a list of Qdrant FieldCondition objects based on params """ - collection_param_map = { - RESOURCES_COLLECTION_NAME: QDRANT_RESOURCE_PARAM_MAP, - TOPICS_COLLECTION_NAME: QDRANT_TOPICS_PARAM_MAP, - CONTENT_FILES_COLLECTION_NAME: QDRANT_CONTENT_FILE_PARAM_MAP, - } - qdrant_param_map = collection_param_map.get(collection_name) + qdrant_param_map = COLLECTION_PARAM_MAP.get(collection_name) if not params or not qdrant_param_map: return None must = [] From 3e1ed7a41856a1d1eb10759a94b907c21a5f7066 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 11:39:24 -0400 Subject: [PATCH 19/33] adding aggregation params for contentfiles --- vector_search/serializers.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index bd1b5a5fbe..cce0f8a44f 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -20,7 +20,10 @@ SearchResponseMetadata, SearchResponseSerializer, ) -from vector_search.constants import QDRANT_RESOURCE_PARAM_MAP +from vector_search.constants import ( + QDRANT_CONTENT_FILE_PARAM_MAP, + QDRANT_RESOURCE_PARAM_MAP, +) class LearningResourcesVectorSearchRequestSerializer(serializers.Serializer): @@ -215,6 +218,17 @@ class ContentFileVectorSearchRequestSerializer(serializers.Serializer): limit = serializers.IntegerField( required=False, help_text="Number of results to return per page" ) + aggregation_choices = [ + (key, key.replace("_", " ").title()) for key in QDRANT_CONTENT_FILE_PARAM_MAP + ] + aggregations = serializers.ListField( + required=False, + child=serializers.ChoiceField(choices=aggregation_choices), + help_text=( + f"aggregations for facet counts \ + \n\n{build_choice_description_list(aggregation_choices)}" + ), + ) sortby = serializers.ChoiceField( required=False, choices=CONTENT_FILE_SORTBY_OPTIONS, From d09c4b5a77d7f7aa2881ebdd8d38abf0bef18692 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 11:45:46 -0400 Subject: [PATCH 20/33] regenerate spec --- frontends/api/src/generated/v0/api.ts | 43 ++++++++++++++++++++++ openapi/specs/v0.yaml | 52 +++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/frontends/api/src/generated/v0/api.ts b/frontends/api/src/generated/v0/api.ts index 84156d2899..8e0eb0472a 100644 --- a/frontends/api/src/generated/v0/api.ts +++ b/frontends/api/src/generated/v0/api.ts @@ -11532,6 +11532,7 @@ export const VectorContentFilesSearchApiAxiosParamCreator = function ( /** * Vector Search for content * @summary Content File Vector Search + * @param {Array} [aggregations] aggregations for facet counts * `key` - Key * `course_number` - Course Number * `platform` - Platform * `offered_by` - Offered By * `file_extension` - File Extension * `content_feature_type` - Content Feature Type * `run_readable_id` - Run Readable Id * `resource_readable_id` - Resource Readable Id * `run_title` - Run Title * `edx_module_id` - Edx Module Id * `content_type` - Content Type * `description` - Description * `title` - Title * `url` - Url * `file_type` - File Type * `summary` - Summary * `flashcards` - Flashcards * `checksum` - Checksum * @param {string} [collection_name] Manually specify the name of the Qdrant collection to query * @param {Array} [file_extension] The extension of the content file. * @param {string} [group_by] The attribute to group results by @@ -11552,6 +11553,7 @@ export const VectorContentFilesSearchApiAxiosParamCreator = function ( * @throws {RequiredError} */ vectorContentFilesSearchRetrieve: async ( + aggregations?: Array, collection_name?: string, file_extension?: Array, group_by?: string, @@ -11586,6 +11588,10 @@ export const VectorContentFilesSearchApiAxiosParamCreator = function ( const localVarHeaderParameter = {} as any const localVarQueryParameter = {} as any + if (aggregations) { + localVarQueryParameter["aggregations"] = aggregations + } + if (collection_name !== undefined) { localVarQueryParameter["collection_name"] = collection_name } @@ -11680,6 +11686,7 @@ export const VectorContentFilesSearchApiFp = function ( /** * Vector Search for content * @summary Content File Vector Search + * @param {Array} [aggregations] aggregations for facet counts * `key` - Key * `course_number` - Course Number * `platform` - Platform * `offered_by` - Offered By * `file_extension` - File Extension * `content_feature_type` - Content Feature Type * `run_readable_id` - Run Readable Id * `resource_readable_id` - Resource Readable Id * `run_title` - Run Title * `edx_module_id` - Edx Module Id * `content_type` - Content Type * `description` - Description * `title` - Title * `url` - Url * `file_type` - File Type * `summary` - Summary * `flashcards` - Flashcards * `checksum` - Checksum * @param {string} [collection_name] Manually specify the name of the Qdrant collection to query * @param {Array} [file_extension] The extension of the content file. * @param {string} [group_by] The attribute to group results by @@ -11700,6 +11707,7 @@ export const VectorContentFilesSearchApiFp = function ( * @throws {RequiredError} */ async vectorContentFilesSearchRetrieve( + aggregations?: Array, collection_name?: string, file_extension?: Array, group_by?: string, @@ -11725,6 +11733,7 @@ export const VectorContentFilesSearchApiFp = function ( > { const localVarAxiosArgs = await localVarAxiosParamCreator.vectorContentFilesSearchRetrieve( + aggregations, collection_name, file_extension, group_by, @@ -11783,6 +11792,7 @@ export const VectorContentFilesSearchApiFactory = function ( ): AxiosPromise { return localVarFp .vectorContentFilesSearchRetrieve( + requestParameters.aggregations, requestParameters.collection_name, requestParameters.file_extension, requestParameters.group_by, @@ -11812,6 +11822,13 @@ export const VectorContentFilesSearchApiFactory = function ( * @interface VectorContentFilesSearchApiVectorContentFilesSearchRetrieveRequest */ export interface VectorContentFilesSearchApiVectorContentFilesSearchRetrieveRequest { + /** + * aggregations for facet counts * `key` - Key * `course_number` - Course Number * `platform` - Platform * `offered_by` - Offered By * `file_extension` - File Extension * `content_feature_type` - Content Feature Type * `run_readable_id` - Run Readable Id * `resource_readable_id` - Resource Readable Id * `run_title` - Run Title * `edx_module_id` - Edx Module Id * `content_type` - Content Type * `description` - Description * `title` - Title * `url` - Url * `file_type` - File Type * `summary` - Summary * `flashcards` - Flashcards * `checksum` - Checksum + * @type {Array<'key' | 'course_number' | 'platform' | 'offered_by' | 'file_extension' | 'content_feature_type' | 'run_readable_id' | 'resource_readable_id' | 'run_title' | 'edx_module_id' | 'content_type' | 'description' | 'title' | 'url' | 'file_type' | 'summary' | 'flashcards' | 'checksum'>} + * @memberof VectorContentFilesSearchApiVectorContentFilesSearchRetrieve + */ + readonly aggregations?: Array + /** * Manually specify the name of the Qdrant collection to query * @type {string} @@ -11946,6 +11963,7 @@ export class VectorContentFilesSearchApi extends BaseAPI { ) { return VectorContentFilesSearchApiFp(this.configuration) .vectorContentFilesSearchRetrieve( + requestParameters.aggregations, requestParameters.collection_name, requestParameters.file_extension, requestParameters.group_by, @@ -11968,6 +11986,31 @@ export class VectorContentFilesSearchApi extends BaseAPI { } } +/** + * @export + */ +export const VectorContentFilesSearchRetrieveAggregationsEnum = { + Key: "key", + CourseNumber: "course_number", + Platform: "platform", + OfferedBy: "offered_by", + FileExtension: "file_extension", + ContentFeatureType: "content_feature_type", + RunReadableId: "run_readable_id", + ResourceReadableId: "resource_readable_id", + RunTitle: "run_title", + EdxModuleId: "edx_module_id", + ContentType: "content_type", + Description: "description", + Title: "title", + Url: "url", + FileType: "file_type", + Summary: "summary", + Flashcards: "flashcards", + Checksum: "checksum", +} as const +export type VectorContentFilesSearchRetrieveAggregationsEnum = + (typeof VectorContentFilesSearchRetrieveAggregationsEnum)[keyof typeof VectorContentFilesSearchRetrieveAggregationsEnum] /** * @export */ diff --git a/openapi/specs/v0.yaml b/openapi/specs/v0.yaml index 3a686ef55d..85bfabe127 100644 --- a/openapi/specs/v0.yaml +++ b/openapi/specs/v0.yaml @@ -827,6 +827,58 @@ paths: description: Vector Search for content summary: Content File Vector Search parameters: + - in: query + name: aggregations + schema: + type: array + items: + enum: + - key + - course_number + - platform + - offered_by + - file_extension + - content_feature_type + - run_readable_id + - resource_readable_id + - run_title + - edx_module_id + - content_type + - description + - title + - url + - file_type + - summary + - flashcards + - checksum + type: string + description: |- + * `key` - Key + * `course_number` - Course Number + * `platform` - Platform + * `offered_by` - Offered By + * `file_extension` - File Extension + * `content_feature_type` - Content Feature Type + * `run_readable_id` - Run Readable Id + * `resource_readable_id` - Resource Readable Id + * `run_title` - Run Title + * `edx_module_id` - Edx Module Id + * `content_type` - Content Type + * `description` - Description + * `title` - Title + * `url` - Url + * `file_type` - File Type + * `summary` - Summary + * `flashcards` - Flashcards + * `checksum` - Checksum + description: "aggregations for facet counts \n\n* `key` - Key\n\ + * `course_number` - Course Number\n* `platform` - Platform\n* `offered_by`\ + \ - Offered By\n* `file_extension` - File Extension\n* `content_feature_type`\ + \ - Content Feature Type\n* `run_readable_id` - Run Readable Id\n* `resource_readable_id`\ + \ - Resource Readable Id\n* `run_title` - Run Title\n* `edx_module_id` -\ + \ Edx Module Id\n* `content_type` - Content Type\n* `description` - Description\n\ + * `title` - Title\n* `url` - Url\n* `file_type` - File Type\n* `summary`\ + \ - Summary\n* `flashcards` - Flashcards\n* `checksum` - Checksum" - in: query name: collection_name schema: From 493fc35aacb537fdcffbfb9c29b5e0d930d8bb75 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 12:26:20 -0400 Subject: [PATCH 21/33] adding fix for hybrid search offset --- vector_search/views.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/vector_search/views.py b/vector_search/views.py index fbca35c888..931734e60c 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -87,12 +87,12 @@ async def dispatch(self, request, *args, **kwargs): self.response = self.finalize_response(request, response, *args, **kwargs) return self.response - async def async_vector_search( # noqa: PLR0913 + async def async_vector_search( # noqa: PLR0913 PLR0915 PLR0912 self, query_string: str, params: dict, limit: int = 10, - offset: int = 10, + offset: int = 0, search_collection=RESOURCES_COLLECTION_NAME, *, hybrid_search: bool = False, @@ -188,14 +188,29 @@ async def async_vector_search( # noqa: PLR0913 result_obj = await client.query_points(**search_params) search_result = result_obj.points else: - scroll_res = await client.scroll( - collection_name=search_collection, - scroll_filter=search_filter, - limit=limit, - offset=offset, - with_vectors=False, - ) - search_result = scroll_res[0] + # Qdrant's scroll API uses a point-ID cursor for `offset`, not a + # numeric skip count. We implement integer offset by consuming + # scroll pages until the desired number of records are skipped. + remaining_to_skip = offset + next_page_offset = None + search_result = [] + while True: + fetch_size = min(remaining_to_skip + limit, 1000) + scroll_res = await client.scroll( + collection_name=search_collection, + scroll_filter=search_filter, + limit=fetch_size, + offset=next_page_offset, + with_vectors=False, + ) + page_points, next_page_offset = scroll_res + if remaining_to_skip > 0: + page_points = page_points[remaining_to_skip:] + remaining_to_skip = 0 + search_result.extend(page_points) + if len(search_result) >= limit or not next_page_offset: + break + search_result = search_result[:limit] if search_collection == RESOURCES_COLLECTION_NAME: hits = await sync_to_async(_resource_vector_hits)(search_result) From c1b848ffe846430a06ead6d75d2f6c003718d346 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 13:20:23 -0400 Subject: [PATCH 22/33] fix tests for new expected response --- vector_search/views_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vector_search/views_test.py b/vector_search/views_test.py index 774ca25436..981fc4371b 100644 --- a/vector_search/views_test.py +++ b/vector_search/views_test.py @@ -14,7 +14,7 @@ def test_vector_search_filters(mocker, client): mock_qdrant = mocker.patch( "qdrant_client.AsyncQdrantClient", return_value=mocker.AsyncMock() )() - mock_qdrant.scroll = mocker.AsyncMock(return_value=[[]]) + mock_qdrant.scroll = mocker.AsyncMock(return_value=([], None)) mock_qdrant.query_points = mocker.AsyncMock() mock_qdrant.query_points_groups = mocker.AsyncMock() mocker.patch( @@ -63,7 +63,7 @@ def test_vector_search_filters_empty_query(mocker, client): mock_qdrant = mocker.patch( "qdrant_client.AsyncQdrantClient", return_value=mocker.AsyncMock() )() - mock_qdrant.scroll = mocker.AsyncMock(return_value=[[]]) + mock_qdrant.scroll = mocker.AsyncMock(return_value=([], None)) mock_qdrant.query_points = mocker.AsyncMock() mock_qdrant.query_points_groups = mocker.AsyncMock() mock_qdrant.count = mocker.AsyncMock(return_value=CountResult(count=10)) @@ -124,7 +124,7 @@ def test_content_file_vector_search_filters( mock_qdrant = mocker.patch( "qdrant_client.AsyncQdrantClient", return_value=mocker.AsyncMock() )() - mock_qdrant.scroll = mocker.AsyncMock(return_value=[[]]) + mock_qdrant.scroll = mocker.AsyncMock(return_value=([], None)) mock_qdrant.query_points = mocker.AsyncMock() mock_qdrant.query_points_groups = mocker.AsyncMock() mocker.patch( @@ -201,7 +201,7 @@ def test_content_file_vector_search_filters_empty_query( mock_qdrant = mocker.patch( "qdrant_client.AsyncQdrantClient", return_value=mocker.AsyncMock() )() - mock_qdrant.scroll = mocker.AsyncMock(return_value=[[]]) + mock_qdrant.scroll = mocker.AsyncMock(return_value=([], None)) mock_qdrant.query_points = mocker.AsyncMock() mock_qdrant.query_points_groups = mocker.AsyncMock() mocker.patch( @@ -255,7 +255,7 @@ def test_content_file_vector_search_filters_custom_collection( "qdrant_client.AsyncQdrantClient", return_value=mocker.AsyncMock() )() custom_collection_name = "foo_bar_collection" - mock_qdrant.scroll = mocker.AsyncMock(return_value=[[]]) + mock_qdrant.scroll = mocker.AsyncMock(return_value=([], None)) mock_qdrant.query_points = mocker.AsyncMock() mock_qdrant.query_points_groups = mocker.AsyncMock() mocker.patch( @@ -300,7 +300,7 @@ def test_content_file_vector_search_group_parameters(mocker, client, django_user )() custom_collection_name = "foo_bar_collection" - mock_qdrant.scroll = mocker.AsyncMock(return_value=[[]]) + mock_qdrant.scroll = mocker.AsyncMock(return_value=([], None)) mock_qdrant.query_points = mocker.AsyncMock() mock_qdrant.query_points_groups = mocker.AsyncMock() mocker.patch( From de908ef4735925390ef76a2d499fba62993479e7 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 13:24:43 -0400 Subject: [PATCH 23/33] fix contentfile metadata --- vector_search/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index cce0f8a44f..e93a277e67 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -312,8 +312,8 @@ def get_count(self, instance) -> int: def get_results(self, instance): return instance["hits"] - def get_metadata(self, *_) -> SearchResponseMetadata: + def get_metadata(self, instance) -> SearchResponseMetadata: return { - "aggregations": [], + "aggregations": instance.get("aggregations", {}), "suggest": [], } From 266836566505c51cba5f84081da3fe063019049b Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 13:29:50 -0400 Subject: [PATCH 24/33] make hits and get_results same for both serializers --- vector_search/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index e93a277e67..0ef7dfcc42 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -197,7 +197,7 @@ def get_results(self, instance): return instance.get("hits", {}) def get_count(self, instance) -> int: - return instance.get("total", {}).get("value") + return instance.get("total", {}).get("value", 0) def get_metadata(self, instance) -> SearchResponseMetadata: return { @@ -306,11 +306,11 @@ class ContentFileVectorSearchResponseSerializer(SearchResponseSerializer): """ def get_count(self, instance) -> int: - return instance["total"]["value"] + return instance.get("total", {}).get("value", 0) @extend_schema_field(ContentFileSerializer(many=True)) def get_results(self, instance): - return instance["hits"] + return instance.get("hits", {}) def get_metadata(self, instance) -> SearchResponseMetadata: return { From a17581b99b3412902cf4aa9c0dbe2fbbe19cd84b Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 13:31:41 -0400 Subject: [PATCH 25/33] fixing skip with relation to offsets --- vector_search/views.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vector_search/views.py b/vector_search/views.py index 931734e60c..2cb5ae403d 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -195,7 +195,7 @@ async def async_vector_search( # noqa: PLR0913 PLR0915 PLR0912 next_page_offset = None search_result = [] while True: - fetch_size = min(remaining_to_skip + limit, 1000) + fetch_size = min(max(remaining_to_skip, limit), 1000) scroll_res = await client.scroll( collection_name=search_collection, scroll_filter=search_filter, @@ -205,8 +205,9 @@ async def async_vector_search( # noqa: PLR0913 PLR0915 PLR0912 ) page_points, next_page_offset = scroll_res if remaining_to_skip > 0: - page_points = page_points[remaining_to_skip:] - remaining_to_skip = 0 + skipped = min(remaining_to_skip, len(page_points)) + page_points = page_points[skipped:] + remaining_to_skip -= skipped search_result.extend(page_points) if len(search_result) >= limit or not next_page_offset: break From 15663a31d522b7493ebb935a5a2320a8adf3a63f Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 13:58:52 -0400 Subject: [PATCH 26/33] tune prefetch multiplier --- main/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main/settings.py b/main/settings.py index 918d8efc4d..135af39745 100644 --- a/main/settings.py +++ b/main/settings.py @@ -820,10 +820,10 @@ def get_all_config_keys(): QDRANT_CLIENT_TIMEOUT = get_int(name="QDRANT_CLIENT_TIMEOUT", default=10) VECTOR_HYBRID_SEARCH_PREFETCH_MULTIPLIER = get_int( - name="VECTOR_HYBRID_SEARCH_PREFETCH_MULTIPLIER", default=20 + name="VECTOR_HYBRID_SEARCH_PREFETCH_MULTIPLIER", default=5 ) VECTOR_HYBRID_SEARCH_PREFETCH_MAX_LIMIT = get_int( - name="VECTOR_HYBRID_SEARCH_PREFETCH_MAX_LIMIT", default=10000 + name="VECTOR_HYBRID_SEARCH_PREFETCH_MAX_LIMIT", default=500 ) # toggle to use requests (default for local) or webdriver which renders js elements EMBEDDINGS_EXTERNAL_FETCH_USE_WEBDRIVER = get_bool( From 7d1c544e33f37626e88a5d40778f113c30368c99 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Tue, 14 Apr 2026 16:18:43 -0400 Subject: [PATCH 27/33] gather count with hits --- vector_search/views.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/vector_search/views.py b/vector_search/views.py index 2cb5ae403d..5335351c46 100644 --- a/vector_search/views.py +++ b/vector_search/views.py @@ -87,7 +87,7 @@ async def dispatch(self, request, *args, **kwargs): self.response = self.finalize_response(request, response, *args, **kwargs) return self.response - async def async_vector_search( # noqa: PLR0913 PLR0915 PLR0912 + async def async_vector_search( # noqa: PLR0913, PLR0915 self, query_string: str, params: dict, @@ -213,13 +213,15 @@ async def async_vector_search( # noqa: PLR0913 PLR0915 PLR0912 break search_result = search_result[:limit] - if search_collection == RESOURCES_COLLECTION_NAME: - hits = await sync_to_async(_resource_vector_hits)(search_result) - else: - hits = await sync_to_async(_content_file_vector_hits)(search_result) + hits_coroutine = ( + sync_to_async(_resource_vector_hits)(search_result) + if search_collection == RESOURCES_COLLECTION_NAME + else sync_to_async(_content_file_vector_hits)(search_result) + ) aggregation_keys = params.get("aggregations") or [] - count_result, aggregations = await asyncio.gather( + hits, count_result, aggregations = await asyncio.gather( + hits_coroutine, client.count( collection_name=search_collection, count_filter=search_filter, From 9a8f7db8f1435e662822f833ec17730fca52b55a Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Wed, 15 Apr 2026 16:03:35 -0400 Subject: [PATCH 28/33] adding fix for fields returned by contentfile endpoint --- vector_search/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vector_search/constants.py b/vector_search/constants.py index cadd81622a..074fa67991 100644 --- a/vector_search/constants.py +++ b/vector_search/constants.py @@ -97,7 +97,7 @@ } -CONTENT_FILES_RETRIEVE_PAYLOAD = ["key", "run_readable_id"] +CONTENT_FILES_RETRIEVE_PAYLOAD = True RESOURCES_RETRIEVE_PAYLOAD = ["readable_id"] From 027114c5dbb290b47e21bb473c93af053e579fce Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Wed, 15 Apr 2026 19:26:10 -0400 Subject: [PATCH 29/33] default hits to list --- vector_search/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index 0ef7dfcc42..6c7377f653 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -194,7 +194,7 @@ class LearningResourcesVectorSearchResponseSerializer(SearchResponseSerializer): @extend_schema_field(LearningResourceSerializer(many=True)) def get_results(self, instance): - return instance.get("hits", {}) + return instance.get("hits", []) def get_count(self, instance) -> int: return instance.get("total", {}).get("value", 0) @@ -310,7 +310,7 @@ def get_count(self, instance) -> int: @extend_schema_field(ContentFileSerializer(many=True)) def get_results(self, instance): - return instance.get("hits", {}) + return instance.get("hits", []) def get_metadata(self, instance) -> SearchResponseMetadata: return { From d6548929c5b47139d5a229cfc5a12665eeefcd5b Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Wed, 15 Apr 2026 22:45:41 -0400 Subject: [PATCH 30/33] Update vector_search/utils.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- vector_search/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vector_search/utils.py b/vector_search/utils.py index 0027445da4..89791ab06e 100644 --- a/vector_search/utils.py +++ b/vector_search/utils.py @@ -872,7 +872,11 @@ def process_batch(docs_batch, summaries_list): def _resource_vector_hits(search_result): - hits = [hit.payload.get("readable_id") for hit in search_result] + hits = [ + readable_id + for readable_id in (hit.payload.get("readable_id") for hit in search_result) + if readable_id + ] """ Always lookup learning resources by readable_id for portability in case we load points from external systems From 13c0593b3037d0f64afffa107f6b52cc9991e8dd Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 16 Apr 2026 10:41:11 -0400 Subject: [PATCH 31/33] restore and update js test for vector hybrid search facet results --- .../app-pages/SearchPage/SearchPage.test.tsx | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx b/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx index 2d9cb44807..028bde95ed 100644 --- a/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx +++ b/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx @@ -1016,4 +1016,61 @@ describe("UniversalAIBanner", () => { expect(screen.queryByText("Universal AI")).not.toBeInTheDocument() expect(screen.queryByText("New on MIT Learn")).not.toBeInTheDocument() }) + + test("Vector Hybrid Search passes correct params and renders expected count/facets", async () => { + setMockApiResponses({ + search: { + count: 700, + metadata: { + aggregations: { + resource_type_group: [{ key: "course", doc_count: 100 }], + }, + suggestions: [], + }, + results: factories.learningResources.resources({ count: 5 }).results, + }, + }) + + // Authenticate as path editor (admin) + setMockResponse.get(urls.userMe.get(), { + is_learning_path_editor: true, + is_authenticated: true, + }) + + renderWithProviders(, { url: "?vector_search=true&q=test" }) + + await waitFor(() => { + const call = makeRequest.mock.calls.find(([_method, url]) => { + return url.includes(urls.search.vectorResources()) + }) + expect(call).toBeDefined() + }) + + const call = makeRequest.mock.calls.find(([_method, url]) => + url.includes(urls.search.vectorResources()), + ) + invariant(call) + const fullUrl = new URL(call[1], "http://mit.edu") + const apiSearchParams = fullUrl.searchParams + + expect(apiSearchParams.get("hybrid_search")).toBe("true") + expect(apiSearchParams.get("q")).toBe("test") + + // Ensure count is visible + const countText = await screen.findByText("700 results") + expect(countText).toBeVisible() + + // Ensure facets are visible + await waitFor(() => { + const tabs = screen.getAllByRole("tab") + expect( + tabs.map((tab) => (tab.textContent || "").replace(/\s/g, "")), + ).toEqual([ + "All(100)", + "Courses(100)", + "Programs(0)", + "LearningMaterials(0)", + ]) + }) + }) }) From 2348970be680f05a214738c97163d36f8ba6dbdb Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 16 Apr 2026 10:45:27 -0400 Subject: [PATCH 32/33] move published to resource specific serializer field --- vector_search/serializers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vector_search/serializers.py b/vector_search/serializers.py index ee6c51f563..dbd5614467 100644 --- a/vector_search/serializers.py +++ b/vector_search/serializers.py @@ -35,11 +35,6 @@ class LearningResourcesSearchFiltersSerializer(serializers.Serializer): to a Qdrant filter by qdrant_query_conditions(). """ - published = serializers.BooleanField( - required=False, - default=True, - help_text="If the resource is published. We default to True unless passed in", - ) offered_by_choices = [(e.name.lower(), e.value) for e in OfferedBy] offered_by = serializers.ListField( required=False, @@ -160,6 +155,11 @@ class LearningResourcesVectorSearchRequestSerializer( instead of id we use readable_id in case we upload qdrant snapshots """ + published = serializers.BooleanField( + required=False, + default=True, + help_text="If the resource is published. We default to True unless passed in", + ) aggregation_choices = [ (key, key.replace("_", " ").title()) for key in QDRANT_RESOURCE_PARAM_MAP ] From 7fa59ddba5637a62d23a0cfe9112fa06a5d897fd Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 16 Apr 2026 10:48:45 -0400 Subject: [PATCH 33/33] update spec --- frontends/api/src/generated/v1/api.ts | 36 --------------------------- openapi/specs/v1.yaml | 14 ----------- 2 files changed, 50 deletions(-) diff --git a/frontends/api/src/generated/v1/api.ts b/frontends/api/src/generated/v1/api.ts index 81d2fccfaf..392613c715 100644 --- a/frontends/api/src/generated/v1/api.ts +++ b/frontends/api/src/generated/v1/api.ts @@ -16458,7 +16458,6 @@ export const LearningResourcesApiAxiosParamCreator = function ( * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `climate` - MIT Climate * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube * `canvas` - Canvas * `climate` - MIT Climate * `ovs` - ODL Video Service * @param {boolean | null} [professional] - * @param {boolean} [published] If the resource is published. We default to True unless passed in * @param {Array} [resource_type] The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document * @param {Array} [resource_type_group] The category of learning resource * `course` - Course * `program` - Program * `learning_material` - Learning Material * @param {Array} [topic] The topic name. To see a list of options go to api/v1/topics/ @@ -16479,7 +16478,6 @@ export const LearningResourcesApiAxiosParamCreator = function ( offered_by?: Array, platform?: Array, professional?: boolean | null, - published?: boolean, resource_type?: Array, resource_type_group?: Array, topic?: Array, @@ -16554,10 +16552,6 @@ export const LearningResourcesApiAxiosParamCreator = function ( localVarQueryParameter["professional"] = professional } - if (published !== undefined) { - localVarQueryParameter["published"] = published - } - if (resource_type) { localVarQueryParameter["resource_type"] = resource_type } @@ -16808,7 +16802,6 @@ export const LearningResourcesApiAxiosParamCreator = function ( * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `climate` - MIT Climate * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube * `canvas` - Canvas * `climate` - MIT Climate * `ovs` - ODL Video Service * @param {boolean | null} [professional] - * @param {boolean} [published] If the resource is published. We default to True unless passed in * @param {Array} [resource_type] The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document * @param {Array} [resource_type_group] The category of learning resource * `course` - Course * `program` - Program * `learning_material` - Learning Material * @param {Array} [topic] The topic name. To see a list of options go to api/v1/topics/ @@ -16829,7 +16822,6 @@ export const LearningResourcesApiAxiosParamCreator = function ( offered_by?: Array, platform?: Array, professional?: boolean | null, - published?: boolean, resource_type?: Array, resource_type_group?: Array, topic?: Array, @@ -16905,10 +16897,6 @@ export const LearningResourcesApiAxiosParamCreator = function ( localVarQueryParameter["professional"] = professional } - if (published !== undefined) { - localVarQueryParameter["published"] = published - } - if (resource_type) { localVarQueryParameter["resource_type"] = resource_type } @@ -17290,7 +17278,6 @@ export const LearningResourcesApiFp = function (configuration?: Configuration) { * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `climate` - MIT Climate * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube * `canvas` - Canvas * `climate` - MIT Climate * `ovs` - ODL Video Service * @param {boolean | null} [professional] - * @param {boolean} [published] If the resource is published. We default to True unless passed in * @param {Array} [resource_type] The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document * @param {Array} [resource_type_group] The category of learning resource * `course` - Course * `program` - Program * `learning_material` - Learning Material * @param {Array} [topic] The topic name. To see a list of options go to api/v1/topics/ @@ -17311,7 +17298,6 @@ export const LearningResourcesApiFp = function (configuration?: Configuration) { offered_by?: Array, platform?: Array, professional?: boolean | null, - published?: boolean, resource_type?: Array, resource_type_group?: Array, topic?: Array, @@ -17337,7 +17323,6 @@ export const LearningResourcesApiFp = function (configuration?: Configuration) { offered_by, platform, professional, - published, resource_type, resource_type_group, topic, @@ -17497,7 +17482,6 @@ export const LearningResourcesApiFp = function (configuration?: Configuration) { * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `climate` - MIT Climate * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube * `canvas` - Canvas * `climate` - MIT Climate * `ovs` - ODL Video Service * @param {boolean | null} [professional] - * @param {boolean} [published] If the resource is published. We default to True unless passed in * @param {Array} [resource_type] The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document * @param {Array} [resource_type_group] The category of learning resource * `course` - Course * `program` - Program * `learning_material` - Learning Material * @param {Array} [topic] The topic name. To see a list of options go to api/v1/topics/ @@ -17518,7 +17502,6 @@ export const LearningResourcesApiFp = function (configuration?: Configuration) { offered_by?: Array, platform?: Array, professional?: boolean | null, - published?: boolean, resource_type?: Array, resource_type_group?: Array, topic?: Array, @@ -17544,7 +17527,6 @@ export const LearningResourcesApiFp = function (configuration?: Configuration) { offered_by, platform, professional, - published, resource_type, resource_type_group, topic, @@ -17758,7 +17740,6 @@ export const LearningResourcesApiFactory = function ( requestParameters.offered_by, requestParameters.platform, requestParameters.professional, - requestParameters.published, requestParameters.resource_type, requestParameters.resource_type_group, requestParameters.topic, @@ -17847,7 +17828,6 @@ export const LearningResourcesApiFactory = function ( requestParameters.offered_by, requestParameters.platform, requestParameters.professional, - requestParameters.published, requestParameters.resource_type, requestParameters.resource_type_group, requestParameters.topic, @@ -18277,13 +18257,6 @@ export interface LearningResourcesApiLearningResourcesSimilarListRequest { */ readonly professional?: boolean | null - /** - * If the resource is published. We default to True unless passed in - * @type {boolean} - * @memberof LearningResourcesApiLearningResourcesSimilarList - */ - readonly published?: boolean - /** * The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document * @type {Array<'course' | 'program' | 'learning_path' | 'podcast' | 'podcast_episode' | 'video' | 'video_playlist' | 'document'>} @@ -18564,13 +18537,6 @@ export interface LearningResourcesApiLearningResourcesVectorSimilarListRequest { */ readonly professional?: boolean | null - /** - * If the resource is published. We default to True unless passed in - * @type {boolean} - * @memberof LearningResourcesApiLearningResourcesVectorSimilarList - */ - readonly published?: boolean - /** * The type of learning resource * `course` - course * `program` - program * `learning_path` - learning path * `podcast` - podcast * `podcast_episode` - podcast episode * `video` - video * `video_playlist` - video playlist * `document` - document * @type {Array<'course' | 'program' | 'learning_path' | 'podcast' | 'podcast_episode' | 'video' | 'video_playlist' | 'document'>} @@ -18796,7 +18762,6 @@ export class LearningResourcesApi extends BaseAPI { requestParameters.offered_by, requestParameters.platform, requestParameters.professional, - requestParameters.published, requestParameters.resource_type, requestParameters.resource_type_group, requestParameters.topic, @@ -18891,7 +18856,6 @@ export class LearningResourcesApi extends BaseAPI { requestParameters.offered_by, requestParameters.platform, requestParameters.professional, - requestParameters.published, requestParameters.resource_type, requestParameters.resource_type_group, requestParameters.topic, diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index d30decec55..c29644cba6 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -2988,13 +2988,6 @@ paths: schema: type: boolean nullable: true - - in: query - name: published - schema: - type: boolean - default: true - description: If the resource is published. We default to True unless passed - in - in: query name: resource_type schema: @@ -3358,13 +3351,6 @@ paths: schema: type: boolean nullable: true - - in: query - name: published - schema: - type: boolean - default: true - description: If the resource is published. We default to True unless passed - in - in: query name: resource_type schema: