Fix course blocklist exclusion to use __in lookup in index/embed tasks

The recreate-index and embed tasks excluded blocklisted courses with an
exact-match lookup against a list (readable_id=blocklisted_ids), which
CharField stringifies into a value that matches nothing — so the
exclusion was a silent no-op. Use readable_id__in like the other call
sites, and add tests that fail against the old lookup.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: mbertrand

learning_resources_search/tasks.py

@@ -624,7 +624,7 @@ def start_recreate_index(self, indexes, remove_existing_reindexing_tags):
                 )
                 for ids in chunks(
                     Course.objects.filter(learning_resource__published=True)
-                    .exclude(learning_resource__readable_id=blocklisted_ids)
+                    .exclude(learning_resource__readable_id__in=blocklisted_ids)
                     .order_by("learning_resource_id")
                     .values_list("learning_resource_id", flat=True),
                     chunk_size=settings.OPENSEARCH_INDEXING_CHUNK_SIZE,
@@ -634,7 +634,7 @@ def start_recreate_index(self, indexes, remove_existing_reindexing_tags):
             for course in (
                 Course.objects.filter(learning_resource__published=True)
                 .filter(learning_resource__etl_source__in=RESOURCE_FILE_ETL_SOURCES)
-                .exclude(learning_resource__readable_id=blocklisted_ids)
+                .exclude(learning_resource__readable_id__in=blocklisted_ids)
                 .order_by("learning_resource_id")
             ):
                 index_tasks = index_tasks + [
@@ -668,7 +668,7 @@ def start_recreate_index(self, indexes, remove_existing_reindexing_tags):
                     LearningResource.objects.filter(
                         published=True,
                     )
-                    .exclude(readable_id=blocklisted_ids)
+                    .exclude(readable_id__in=blocklisted_ids)
                     .order_by("id")
                     .values_list("id", flat=True),
                     chunk_size=settings.OPENSEARCH_INDEXING_CHUNK_SIZE,

learning_resources_search/tasks_test.py

@@ -285,6 +285,59 @@ def test_start_recreate_index(mocker, mocked_celery, user, indexes):  # noqa: C9
     assert mocked_celery.replace.call_args[0][1] == mocked_celery.chain.return_value
 
 
+@pytest.mark.parametrize("indexes", [["course"], ["combined_hybrid"]])
+def test_start_recreate_index_excludes_blocklisted_courses(
+    mocker, mocked_celery, indexes
+):
+    """start_recreate_index should not index courses whose readable_id is blocklisted"""
+    courses = CourseFactory.create_batch(3, etl_source=ETLSource.ocw.value)
+    blocked = courses[0]
+    for course in courses:
+        ContentFileFactory.create_batch(2, run=course.learning_resource.runs.first())
+    mocker.patch(
+        "learning_resources_search.tasks.load_course_blocklist",
+        return_value=[blocked.learning_resource.readable_id],
+    )
+    index_learning_resources_mock = mocker.patch(
+        "learning_resources_search.tasks.index_learning_resources", autospec=True
+    )
+    index_files_mock = mocker.patch(
+        "learning_resources_search.tasks.index_content_files", autospec=True
+    )
+    mocker.patch(
+        "learning_resources_search.indexing_api.create_backing_index",
+        autospec=True,
+        return_value="backing",
+    )
+    mocker.patch(
+        "learning_resources_search.indexing_api.get_existing_reindexing_indexes",
+        autospec=True,
+        return_value=[],
+    )
+    mocker.patch(
+        "learning_resources_search.indexing_api.delete_orphaned_indexes", autospec=True
+    )
+    mocker.patch("learning_resources_search.tasks.finish_recreate_index", autospec=True)
+
+    with pytest.raises(mocked_celery.replace_exception_class):
+        start_recreate_index.delay(indexes, remove_existing_reindexing_tags=False)
+
+    indexed_resource_ids = {
+        resource_id
+        for call in index_learning_resources_mock.si.call_args_list
+        for resource_id in call.args[0]
+    }
+    assert indexed_resource_ids == {
+        course.learning_resource_id for course in courses[1:]
+    }
+
+    if COURSE_TYPE in indexes:
+        file_course_ids = {call.args[1] for call in index_files_mock.si.call_args_list}
+        assert file_course_ids == {
+            course.learning_resource_id for course in courses[1:]
+        }
+
+
 @pytest.mark.parametrize(
     "remove_existing_reindexing_tags",
     [True, False],

vector_search/tasks.py

@@ -248,7 +248,7 @@ def start_embed_resources(self, indexes, skip_content_files, overwrite):  # noqa
                 generate_embeddings.si(ids, COURSE_TYPE, overwrite)
                 for ids in chunks(
                     Course.objects.filter(learning_resource__published=True)
-                    .exclude(learning_resource__readable_id=blocklisted_ids)
+                    .exclude(learning_resource__readable_id__in=blocklisted_ids)
                     .order_by("learning_resource_id")
                     .values_list("learning_resource_id", flat=True),
                     chunk_size=settings.QDRANT_CHUNK_SIZE,
@@ -261,7 +261,7 @@ def start_embed_resources(self, indexes, skip_content_files, overwrite):  # noqa
                         resource_type=COURSE_TYPE,
                     )
                     .filter(Q(published=True) | Q(test_mode=True))
-                    .exclude(readable_id=blocklisted_ids)
+                    .exclude(readable_id__in=blocklisted_ids)
                     .order_by("id")
                 ):
                     # Embed published content files across all runs of the course

vector_search/tasks_test.py

@@ -146,6 +146,50 @@ def test_start_embed_resources_without_settings(mocker, mocked_celery, index):
     generate_embeddings_mock.si.assert_not_called()
 
 
+def test_start_embed_resources_excludes_blocklisted_courses(mocker, mocked_celery):
+    """start_embed_resources should not embed courses whose readable_id is blocklisted"""
+    courses = CourseFactory.create_batch(3, etl_source=ETLSource.ocw.value)
+    blocked = courses[0]
+    for course in courses:
+        ContentFileFactory.create_batch(2, run=course.learning_resource.runs.first())
+    mocker.patch(
+        "vector_search.tasks.load_course_blocklist",
+        return_value=[blocked.learning_resource.readable_id],
+    )
+    generate_embeddings_mock = mocker.patch(
+        "vector_search.tasks.generate_embeddings", autospec=True
+    )
+
+    with pytest.raises(mocked_celery.replace_exception_class):
+        start_embed_resources.delay(
+            [COURSE_TYPE], skip_content_files=False, overwrite=True
+        )
+
+    embedded_resource_ids = {
+        resource_id
+        for call in generate_embeddings_mock.si.call_args_list
+        if call.args[1] == COURSE_TYPE
+        for resource_id in call.args[0]
+    }
+    assert embedded_resource_ids == {
+        course.learning_resource_id for course in courses[1:]
+    }
+
+    blocked_file_ids = set(
+        ContentFile.objects.filter(
+            run__learning_resource=blocked.learning_resource
+        ).values_list("id", flat=True)
+    )
+    embedded_file_ids = {
+        file_id
+        for call in generate_embeddings_mock.si.call_args_list
+        if call.args[1] == CONTENT_FILE_TYPE
+        for file_id in call.args[0]
+    }
+    assert embedded_file_ids
+    assert not embedded_file_ids & blocked_file_ids
+
+
 def test_embed_new_learning_resources(mocker, mocked_celery):
     """
     embed_new_learning_resources should generate embeddings for new resources