remove duplicate resource task (#3436)

Author: shanbady

learning_resources/tasks.py

@@ -11,7 +11,7 @@
 from celery.exceptions import Ignore
 from django.conf import settings
 from django.db import OperationalError
-from django.db.models import Count, Q
+from django.db.models import Q
 from django.utils import timezone
 
 from learning_resources.constants import LearningResourceType
@@ -51,7 +51,6 @@
 )
 from learning_resources_search.constants import (
     CONTENT_FILE_TYPE,
-    COURSE_TYPE,
     SEARCH_CONN_EXCEPTIONS,
 )
 from learning_resources_search.exceptions import RetryError
@@ -65,36 +64,6 @@
 CLEANUP_RETRY_EXCEPTIONS = (*SEARCH_CONN_EXCEPTIONS, OperationalError)
 
 
-@app.task(bind=True)
-def remove_duplicate_resources(self):
-    """Remove duplicate unpublished resources"""
-    from vector_search.tasks import generate_embeddings
-
-    duplicates = (
-        LearningResource.objects.values("readable_id")
-        .annotate(count_id=Count("id"))
-        .filter(count_id__gt=1)
-    )
-    embed_tasks = []
-    for duplicate in duplicates:
-        unpublished_resources = LearningResource.objects.filter(
-            readable_id=duplicate["readable_id"],
-            published=False,
-        ).values_list("id", flat=True)
-        published_resources = list(
-            LearningResource.objects.filter(
-                readable_id=duplicate["readable_id"],
-                published=True,
-            ).values_list("id", flat=True)
-        )
-        # keep the most recently created resource, delete the rest
-        LearningResource.objects.filter(id__in=unpublished_resources).delete()
-        embed_tasks.append(
-            generate_embeddings.si(published_resources, COURSE_TYPE, overwrite=True)
-        )
-    self.replace(celery.chain(*embed_tasks))
-
-
 @app.task
 def update_next_start_date_and_prices():
     """Update expired next start dates and prices"""

learning_resources/tasks_test.py

@@ -17,7 +17,6 @@
 from learning_resources.factories import (
     ContentFileFactory,
     LearningResourceFactory,
-    LearningResourcePlatformFactory,
     LearningResourceRunFactory,
 )
 from learning_resources.models import ContentFile, LearningResource
@@ -27,7 +26,6 @@
     get_youtube_data,
     get_youtube_transcripts,
     marketing_page_for_resources,
-    remove_duplicate_resources,
     scrape_marketing_pages,
     sync_canvas_courses,
     update_next_start_date_and_prices,
@@ -839,37 +837,6 @@ def test_sync_canvas_courses(settings, mocker, django_assert_num_queries, canvas
         assert mock_ingest_course.call_count == 2
 
 
-def test_remove_duplicate_resources(mocker, mocked_celery):
-    """
-    Test that remove_duplicate_resources removes duplicate unpublished resources
-    while keeping the most recently created resource.
-    """
-    duplicate_id = "duplicate_id"
-
-    for platform_type in [PlatformType.edx, PlatformType.xpro, PlatformType.youtube]:
-        LearningResourceFactory.create(
-            readable_id=duplicate_id,
-            published=False,
-            platform=LearningResourcePlatformFactory.create(code=platform_type.name),
-        )
-
-    published_reasource = LearningResourceFactory.create(
-        readable_id=duplicate_id,
-        published=True,
-        platform=LearningResourcePlatformFactory.create(
-            code=platform_type.mitxonline.name
-        ),
-    )
-    generate_embeddings_mock = mocker.patch(
-        "vector_search.tasks.generate_embeddings", autospec=True
-    )
-    assert LearningResource.objects.filter(readable_id=duplicate_id).count() == 4
-    with pytest.raises(mocked_celery.replace_exception_class):
-        remove_duplicate_resources()
-    assert generate_embeddings_mock.mock_calls[0].args[0] == [published_reasource.id]
-    assert LearningResource.objects.filter(readable_id=duplicate_id).count() == 1
-
-
 @pytest.mark.parametrize(
     ("etl_source", "archive_path", "overwrite"),
     [

main/settings_celery.py

@@ -169,10 +169,6 @@
                 "SCRAPE_MARKETING_PAGES_SCHEDULE_SECONDS", 60 * 60 * 12
             ),  # default is every 12 hours
         },
-        "remove-duplicate-courses-every-6-hours": {
-            "task": "learning_resources.tasks.remove_duplicate_resources",
-            "schedule": crontab(minute=0, hour=9),  # 5:00am EST
-        },
         "daily_embed_new_learning_resources": {
             "task": "vector_search.tasks.embed_new_learning_resources",
             "schedule": get_int(