Add setting to allow admins to avoid timeout issues with immediate tasks

[PULP-1299]

(cherry picked from commit 71c6e2c)

Author: gerrod3

CHANGES/+task_immediate_timeout.bugfix

@@ -0,0 +1 @@
+Added new setting `TASK_PREFER_DEFER` to always defer immediate tasks to a task worker. Useful for systems with slow databases that frequently timeout immediate tasks.

docs/admin/reference/settings.md

@@ -392,6 +392,16 @@ This time is only accurate to one worker heartbeat corresponding to `WORKER_TTL
 
 Defaults to `600` seconds.
 
+### TASK\_PREFER\_DEFER
+
+For tasks that are designed to run immediately, but could be deferred to a task worker, always choose to defer to the worker.
+This is helpful for systems with slow databases that could cause the immediate task to timeout after its 5 second limit.
+
+!!! note
+    Setting this to `True` will slow your Pulp's task throughput, especially if you perform many immediate tasks frequently.
+
+Defaults to `False`.
+
 ### TASK\_PROTECTION\_TIME, TMPFILE\_PROTECTION\_TIME and UPLOAD\_PROTECTION\_TIME
 
 Pulp uses `tasks`, `pulp temporary files` and `uploads` to pass data from the api to worker tasks.

pulpcore/app/settings.py

@@ -391,6 +391,9 @@
 # NOTE: "memray" and "pyinstrument" require additional packages to be installed on the system.
 TASK_DIAGNOSTICS = []  # ["memory", "pyinstrument", "memray", "logs", "debug-logs"]
 
+# For immediate tasks that can be deferred, always defer them to a worker.
+TASK_PREFER_DEFER = False
+
 ANALYTICS = True
 
 HIDE_GUARDED_DISTRIBUTIONS = False

pulpcore/tasking/tasks.py

@@ -289,6 +289,8 @@ def dispatch(
     Raises:
         ValueError: When `resources` is an unsupported type.
     """
+    if settings.TASK_PREFER_DEFER and deferred and immediate:
+        immediate = False
     # Check WORKER_TYPE setting and delegate to appropriate implementation
     if settings.WORKER_TYPE == "redis":
         from pulpcore.tasking.redis_tasks import dispatch as redis_dispatch
@@ -347,6 +349,8 @@ async def adispatch(
     versions=None,
 ):
     """Async version of dispatch."""
+    if settings.TASK_PREFER_DEFER and deferred and immediate:
+        immediate = False
     # Check WORKER_TYPE setting and delegate to appropriate implementation
     if settings.WORKER_TYPE == "redis":
         from pulpcore.tasking.redis_tasks import adispatch as redis_adispatch