Merge branch 'master' into PROD-4437/edx-ingest-getsmarter-data-cron-job

Author: skumargupta83

.ci/docker-compose-ci.yml

@@ -29,11 +29,12 @@ services:
     container_name: memcached
   discovery:
     # Uncomment this line to use the official course-discovery base image
-    build:
-      context: ../.
-      target: dev
-      args:
-        PYTHON_VERSION: "${PYTHON_VERSION}"
+    # build:
+    #   context: ../.
+    #   target: dev
+    #   args:
+    #     PYTHON_VERSION: "${PYTHON_VERSION}"
+    image: edxops/discovery-dev:latest
     # Uncomment the next two lines to build from a local configuration repo
     # build: ../configuration/docker/build/discovery/
 

.dockerignore

@@ -45,8 +45,8 @@
 from course_discovery.apps.course_metadata.utils import (
     calculated_seat_upgrade_deadline, clean_html, convert_svg_to_png_from_url, create_missing_entitlement,
     download_and_save_course_image, download_and_save_program_image, ensure_draft_world, fetch_getsmarter_products,
-    generate_sku, is_google_drive_url, serialize_entitlement_for_ecommerce_api, serialize_seat_for_ecommerce_api,
-    transform_skills_data, validate_slug_format, is_fatal_error
+    generate_sku, is_fatal_error, is_google_drive_url, serialize_entitlement_for_ecommerce_api,
+    serialize_seat_for_ecommerce_api, transform_skills_data, validate_slug_format
 )
 
 
@@ -1173,6 +1173,49 @@ def test_is_fatal_code(self):
         assert not is_fatal_error(HttpClientError(response=response_with_429))
         assert not is_fatal_error(HttpClientError(response=response_with_504))
 
+    def test_is_fatal_code_error(self):
+        # Success codes (2xx) should not be fatal error
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=201)))
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=204)))
+
+        # Redirect codes (3xx) should not be fatal error
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=301)))
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=302)))
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=307)))
+
+        # Retryable client error (429 Too Many Requests) already covered
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=429)))
+
+        # Retryable server errors (5xx) should not be fatal
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=500)))
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=502)))
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=503)))
+        assert not is_fatal_error(HttpClientError(response=HttpResponse(status=504)))
+
+    def test_is_fatal_code_positive(self):
+        # Client errors (4xx) that should be fatal
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=400)))  # Bad Request
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=401)))  # Unauthorized
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=403)))  # Forbidden
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=404)))  # Not Found
+
+        # Other 4xx codes
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=410)))  # Gone
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=422)))  # Unprocessable Entity
+    
+    def test_is_fatal_code_additional(self):
+        # More client errors (4xx) that should be fatal
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=405)))  # Method Not Allowed
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=406)))  # Not Acceptable
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=407)))  # Proxy Authentication Required
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=408)))  # Request Timeout
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=409)))  # Conflict
+
+        # Validation-related errors
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=415)))  # Unsupported Media Type
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=417)))  # Expectation Failed
+        assert is_fatal_error(HttpClientError(response=HttpResponse(status=451)))  # Unavailable For Legal Reason
+
     def mock_product_api_call(self):
         """
         Mock product api with success response.
@@ -1208,7 +1251,8 @@ def test_fetch_getsmarter_products__with_invalid_credentials(self, mock_getsmart
         products = fetch_getsmarter_products()
         mock_logger.assert_called_with(f'Failed to retrieve products from getsmarter API: {exception_message}')
         assert products == []
-               
+
+
 @ddt.ddt
 class CourseSlugMethodsTests(TestCase):
     """

Dockerfile

@@ -1,4 +1,3 @@
-import backoff
 import datetime
 import logging
 import random
@@ -9,6 +8,7 @@
 from tempfile import NamedTemporaryFile
 from urllib.parse import urljoin, urlparse
 
+import backoff
 import html2text
 import jsonschema
 import markdown
@@ -1014,7 +1014,7 @@ def transform_skills_data(skills_data):
 # any requests that failed in both limits are still approaching their max_tries limit.
 @backoff.on_exception(
     backoff.expo,
-    (requests.exceptions.Timeout,requests.exceptions.HTTPError),
+    (requests.exceptions.Timeout, requests.exceptions.HTTPError),
     factor=60,
     max_tries=4,
     giveup=is_fatal_error,