Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed

- **`derive_image_tag()` prefix** — honours `project.native_image_prefix` instead of hardcoding `capy-`, completing the generic-profile default behaviour documented for Week 30.
- **`PriorityJobQueue.is_done`** — acquires `self._lock`, fixing a torn-read race when the dispatch loop polls completion during concurrent enqueue/complete.

## [0.1.0] - TBD

Expand Down
3 changes: 2 additions & 1 deletion cli/localci/core/queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,8 @@ def is_empty(self) -> bool:

@property
def is_done(self) -> bool:
return len(self._jobs) > 0 and len(self._completed_keys) >= len(self._jobs)
with self._lock:
return len(self._jobs) > 0 and len(self._completed_keys) >= len(self._jobs)

@property
def total_jobs(self) -> int:
Expand Down
74 changes: 74 additions & 0 deletions cli/tests/test_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from localci.core.models import (
JobEventType,
QueuedJob,
QueuedJobStatus,
)
from localci.core.queue import (
CyclicDependencyError,
Expand Down Expand Up @@ -322,6 +323,17 @@ def test_from_config_reads_priorities(self):
# Thread safety
# ---------------------------------------------------------------------------

_TERMINAL_STATUSES = frozenset(
{
QueuedJobStatus.PASSED,
QueuedJobStatus.FAILED,
QueuedJobStatus.CANCELLED,
QueuedJobStatus.SKIPPED,
QueuedJobStatus.ERROR,
QueuedJobStatus.TIMEOUT,
}
)


class TestThreadSafety:
def test_concurrent_enqueue(self):
Expand All @@ -341,6 +353,68 @@ def enqueue_batch(start: int, count: int) -> None:
t.join()
assert queue.total_jobs == 100

def test_concurrent_is_done_invariant(self):

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test_queue.py:356, CHANGELOG.md:28 - the test is not a regression guard.
Give it a failure mode or rename it, and reword the changelog as a consistency fix rather than a fixed race
(with the lock reverted it passed 20 out of 20 runs across 699 observations of is_done == True.
The unlocked read can only go stale in the harmless direction, so a future revert goes unnoticed)

queue = PriorityJobQueue()
stop = threading.Event()
producers_done = threading.Event()
violations: list[str] = []
violations_lock = threading.Lock()

def record(msg: str) -> None:
with violations_lock:
violations.append(msg)

def monitor() -> None:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

read is_done and the job statuses in one locked call, instead of calling is_done and then inspecting what get_all_jobs returned
(get_all_jobs hands back live job objects, so the monitor checks status after the lock is gone and any transient has already healed. Only worth doing if the test can fail at all, since it means adding a production accessor for one test)

while not stop.is_set():
if queue.is_done:
jobs = queue.get_all_jobs()
if not queue.is_done:
time.sleep(0.001)
continue
if not jobs:
record("is_done True with empty queue")
elif not all(j.status in _TERMINAL_STATUSES for j in jobs):
record("is_done True with non-terminal jobs")
time.sleep(0.001)
Comment thread
coderabbitai[bot] marked this conversation as resolved.

def enqueue_batch(start: int, count: int) -> None:
for i in range(start, start + count):
queue.enqueue(make_job(f"Job {i}", priority=1, index=i))
time.sleep(0.001)

def consume() -> None:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

put a deadline on the consumer loop, give every join a timeout, and set timeout-minutes on the ci.yml test job
(the loop exits only when is_done goes true and the joins are unbounded. pytest-timeout is not installed and only the integration job sets a timeout, so finding 1 burns the 6-hour Actions default on three Python versions instead of failing)

while not (producers_done.is_set() and queue.is_done):
job = queue.next_ready()
if job is None:
time.sleep(0.001)
continue
queue.mark_running(job)
queue.mark_completed(job, success=True)

producers = [
threading.Thread(target=enqueue_batch, args=(0, 50)),
threading.Thread(target=enqueue_batch, args=(50, 50)),
]
consumers = [threading.Thread(target=consume) for _ in range(4)]
monitor_thread = threading.Thread(target=monitor)

monitor_thread.start()
for t in producers:
t.start()
for t in consumers:
t.start()

for t in producers:
t.join()
producers_done.set()
for t in consumers:
t.join()
stop.set()
monitor_thread.join()

assert not violations, violations

@clean6378-max-it clean6378-max-it Jul 30, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

assert total_jobs == 100 and passed_count == 100, and have the monitor count it's samples with an assertion
That the count is above zero

(neither assertion pins the job count or shows thee monitor ran , and assert queue.js_done is Trus just restates the consumer loop's own exit condition, so a dead producer leaves the test green)

assert queue.is_done is True

def test_concurrent_dequeue(self):
queue = PriorityJobQueue()
for i in range(20):
Expand Down
Loading