-
Notifications
You must be signed in to change notification settings - Fork 2
fix: take lock in PriorityJobQueue.is_done #101
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,7 @@ | |
| from localci.core.models import ( | ||
| JobEventType, | ||
| QueuedJob, | ||
| QueuedJobStatus, | ||
| ) | ||
| from localci.core.queue import ( | ||
| CyclicDependencyError, | ||
|
|
@@ -322,6 +323,17 @@ def test_from_config_reads_priorities(self): | |
| # Thread safety | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| _TERMINAL_STATUSES = frozenset( | ||
| { | ||
| QueuedJobStatus.PASSED, | ||
| QueuedJobStatus.FAILED, | ||
| QueuedJobStatus.CANCELLED, | ||
| QueuedJobStatus.SKIPPED, | ||
| QueuedJobStatus.ERROR, | ||
| QueuedJobStatus.TIMEOUT, | ||
| } | ||
| ) | ||
|
|
||
|
|
||
| class TestThreadSafety: | ||
| def test_concurrent_enqueue(self): | ||
|
|
@@ -341,6 +353,68 @@ def enqueue_batch(start: int, count: int) -> None: | |
| t.join() | ||
| assert queue.total_jobs == 100 | ||
|
|
||
| def test_concurrent_is_done_invariant(self): | ||
| queue = PriorityJobQueue() | ||
| stop = threading.Event() | ||
| producers_done = threading.Event() | ||
| violations: list[str] = [] | ||
| violations_lock = threading.Lock() | ||
|
|
||
| def record(msg: str) -> None: | ||
| with violations_lock: | ||
| violations.append(msg) | ||
|
|
||
| def monitor() -> None: | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. read is_done and the job statuses in one locked call, instead of calling is_done and then inspecting what get_all_jobs returned |
||
| while not stop.is_set(): | ||
| if queue.is_done: | ||
| jobs = queue.get_all_jobs() | ||
| if not queue.is_done: | ||
| time.sleep(0.001) | ||
| continue | ||
| if not jobs: | ||
| record("is_done True with empty queue") | ||
| elif not all(j.status in _TERMINAL_STATUSES for j in jobs): | ||
| record("is_done True with non-terminal jobs") | ||
| time.sleep(0.001) | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
|
|
||
| def enqueue_batch(start: int, count: int) -> None: | ||
| for i in range(start, start + count): | ||
| queue.enqueue(make_job(f"Job {i}", priority=1, index=i)) | ||
| time.sleep(0.001) | ||
|
|
||
| def consume() -> None: | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. put a deadline on the consumer loop, give every join a timeout, and set timeout-minutes on the ci.yml test job |
||
| while not (producers_done.is_set() and queue.is_done): | ||
| job = queue.next_ready() | ||
| if job is None: | ||
| time.sleep(0.001) | ||
| continue | ||
| queue.mark_running(job) | ||
| queue.mark_completed(job, success=True) | ||
|
|
||
| producers = [ | ||
| threading.Thread(target=enqueue_batch, args=(0, 50)), | ||
| threading.Thread(target=enqueue_batch, args=(50, 50)), | ||
| ] | ||
| consumers = [threading.Thread(target=consume) for _ in range(4)] | ||
| monitor_thread = threading.Thread(target=monitor) | ||
|
|
||
| monitor_thread.start() | ||
| for t in producers: | ||
| t.start() | ||
| for t in consumers: | ||
| t.start() | ||
|
|
||
| for t in producers: | ||
| t.join() | ||
| producers_done.set() | ||
| for t in consumers: | ||
| t.join() | ||
| stop.set() | ||
| monitor_thread.join() | ||
|
|
||
| assert not violations, violations | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. assert total_jobs == 100 and passed_count == 100, and have the monitor count it's samples with an assertion (neither assertion pins the job count or shows thee monitor ran , and assert queue.js_done is Trus just restates the consumer loop's own exit condition, so a dead producer leaves the test green) |
||
| assert queue.is_done is True | ||
|
|
||
| def test_concurrent_dequeue(self): | ||
| queue = PriorityJobQueue() | ||
| for i in range(20): | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
test_queue.py:356, CHANGELOG.md:28 - the test is not a regression guard.
Give it a failure mode or rename it, and reword the changelog as a consistency fix rather than a fixed race
(with the lock reverted it passed 20 out of 20 runs across 699 observations of is_done == True.
The unlocked read can only go stale in the harmless direction, so a future revert goes unnoticed)