fix: ensure navigation sidebar serves fresh data after course publish (#38880)

Replace course_version in the sidebar cache key with a
block_structure_version derived from BlockStructureModel.data_version.
course_version changes eagerly on publish, causing a cache miss before
the block structure is rebuilt — poisoning the cache with stale data for
1 hour. block_structure_version only changes after the async rebuild
completes, so cache misses only occur when fresh data is available.

This is a backport of #38785.

(cherry picked from commit 929815f)

Co-authored-by: Taylor Payne <wgu.taylor.payne@gmail.com>

Author: holaontiveros

lms/djangoapps/course_home_api/outline/tests/test_view.py

@@ -23,6 +23,7 @@
 from lms.djangoapps.course_home_api.toggles import COURSE_HOME_SEND_COURSE_PROGRESS_ANALYTICS_FOR_STUDENT
 from lms.djangoapps.course_home_api.tests.utils import BaseCourseHomeTests
 from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
+from openedx.core.djangoapps.content.block_structure.api import update_course_in_cache
 from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
 from openedx.core.djangoapps.content.learning_sequences.api import replace_course_outline
 from openedx.core.djangoapps.content.learning_sequences.data import CourseOutlineData, CourseVisibility
@@ -900,3 +901,55 @@ def test_vertical_icon_determined_by_icon_class(self):
         response = self.client.get(reverse('course-home:course-navigation', args=[self.course.id]))
         vertical_data = response.data['blocks'][str(self.vertical.location)]
         assert vertical_data['icon'] == 'video'
+
+    def test_navigation_does_not_cache_stale_data_after_publish(self):
+        """
+        Regression test: after the block structure rebuild task completes,
+        the navigation sidebar should serve fresh data.
+
+        This simulates a production scenario where:
+        1. A unit is deleted and the course is auto-published
+        2. The block structure rebuild Celery task is queued with a delay (30s by default)
+        3. A learner hits the navigation endpoint during that 30s window
+        4. The rebuild task completes (bumping block_structure_version)
+        5. Another request arrives
+
+        Without the fix, step 3 caches stale data under a key that step 5
+        also hits (because course_version changed eagerly). With the fix,
+        the cache key uses block_structure_version which only changes when
+        the rebuild completes, so step 5 gets a cache miss and fresh data.
+        """
+        self.add_blocks_to_course()
+        CourseEnrollment.enroll(self.user, self.course.id, CourseMode.VERIFIED)
+
+        # First request — populates both block structure and navigation cache
+        response = self.client.get(self.url)
+        assert response.status_code == 200
+        sequential_data = response.data['blocks'][str(self.sequential.location)]
+        assert str(self.vertical.location) in sequential_data['children']
+
+        # Delete the vertical directly in the modulestore. Signals are disabled
+        # in ModuleStoreTestCase, so the block structure cache is now stale —
+        # mirroring the 30s window in production before the rebuild task runs.
+        self.store.delete_item(self.vertical.location, self.user.id)
+        update_outline_from_modulestore(self.course.id)
+
+        # Request during the stale window — served from the pre-delete cache
+        # (block_structure_version hasn't changed yet, so same cache key).
+        response = self.client.get(self.url)
+        assert response.status_code == 200
+
+        # The vertical is still in the cache, even though it has been deleted
+        sequential_data = response.data['blocks'][str(self.sequential.location)]
+        assert str(self.vertical.location) in sequential_data['children']
+
+        # Now simulate the block structure rebuild task completing.
+        # This bumps block_structure_version → new cache key on next request.
+        update_course_in_cache(self.course.id)
+
+        # Next request has a new cache key (version bumped) → cache miss →
+        # fresh data built from updated block structure.
+        response = self.client.get(self.url)
+        assert response.status_code == 200
+        sequential_data = response.data['blocks'][str(self.sequential.location)]
+        assert str(self.vertical.location) not in sequential_data['children']

lms/djangoapps/course_home_api/outline/views.py

@@ -46,6 +46,7 @@
 from lms.djangoapps.courseware.views.views import get_cert_data
 from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
 from lms.djangoapps.utils import OptimizelyClient
+from openedx.core.djangoapps.content.block_structure.api import get_block_structure_version
 from openedx.core.djangoapps.content.learning_sequences.api import get_user_course_outline
 from openedx.core.djangoapps.content.course_overviews.api import get_course_overview_or_404
 from openedx.core.djangoapps.course_groups.cohorts import get_cohort
@@ -423,7 +424,7 @@ class CourseNavigationBlocksView(RetrieveAPIView):
 
     serializer_class = CourseBlockSerializer
     COURSE_BLOCKS_CACHE_KEY_TEMPLATE = (
-        'course_sidebar_blocks_{course_key_string}_{course_version}_{user_id}_{user_cohort_id}'
+        'course_sidebar_blocks_{course_key_string}_{block_structure_version}_{user_id}_{user_cohort_id}'
         '_{enrollment_mode}_{allow_public}_{allow_public_outline}_{is_masquerading}'
     )
     COURSE_BLOCKS_CACHE_TIMEOUT = 60 * 60  # 1 hour
@@ -458,7 +459,7 @@ def get(self, request, *args, **kwargs):
 
         cache_key = self.COURSE_BLOCKS_CACHE_KEY_TEMPLATE.format(
             course_key_string=course_key_string,
-            course_version=str(course.course_version),
+            block_structure_version=get_block_structure_version(course_key),
             user_id=request.user.id,
             enrollment_mode=getattr(enrollment, 'mode', ''),
             user_cohort_id=getattr(user_cohort, 'id', ''),

openedx/core/djangoapps/content/block_structure/api.py

@@ -8,6 +8,47 @@
 from xmodule.modulestore.django import modulestore
 
 from .manager import BlockStructureManager
+from .models import BlockStructureModel
+
+BLOCK_STRUCTURE_VERSION_KEY = 'block_structure_version:{}'
+
+
+def get_block_structure_version(course_key):
+    """
+    Returns the current block structure version for the given course.
+    This version corresponds to the data_version stored in BlockStructureModel
+    and changes each time the block structure cache is rebuilt.
+
+    Reads from cache first; on a miss, falls back to the database
+    without populating the cache. The cache is populated exclusively by
+    _update_block_structure_version after a successful rebuild, which
+    prevents readers from accidentally caching a stale version during
+    a concurrent rebuild.
+    """
+    cache_key = BLOCK_STRUCTURE_VERSION_KEY.format(course_key)
+    version = cache.get(cache_key)
+    if version is None:
+        try:
+            course_usage_key = modulestore().make_course_usage_key(course_key)
+            block_structure_model = BlockStructureModel.objects.get(data_usage_key=course_usage_key)
+            version = str(block_structure_model.data_version or '')
+        except BlockStructureModel.DoesNotExist:
+            version = ''
+    return version
+
+
+def _update_block_structure_version(course_key):
+    """
+    Reads the current data_version from BlockStructureModel and updates
+    the cached block structure version key.
+    """
+    try:
+        course_usage_key = modulestore().make_course_usage_key(course_key)
+        block_structure_model = BlockStructureModel.objects.get(data_usage_key=course_usage_key)
+        version = str(block_structure_model.data_version or '')
+    except BlockStructureModel.DoesNotExist:
+        version = ''
+    cache.set(BLOCK_STRUCTURE_VERSION_KEY.format(course_key), version, timeout=None)
 
 
 def get_course_in_cache(course_key):
@@ -29,7 +70,8 @@ def update_course_in_cache(course_key):
     block_structure.updated_collected function that updates the block
     structure in the cache for the given course_key.
     """
-    return get_block_structure_manager(course_key).update_collected_if_needed()
+    get_block_structure_manager(course_key).update_collected_if_needed()
+    _update_block_structure_version(course_key)
 
 
 def clear_course_from_cache(course_key):