Skip to content

Commit 7dbb3c5

Browse files
logging improvements and version bump (#79)
* logging improvements and version bump * fix mypy issue * fix tty print issue
1 parent f1b0d09 commit 7dbb3c5

8 files changed

Lines changed: 381 additions & 159 deletions

File tree

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ Each analysis generates an HTML report documenting annotation decisions, reviewe
105105
<img width="1000" alt="CyteType HTML report showing cell type annotations marker genes" src="https://github.com/user-attachments/assets/e5373fdd-7173-42db-b863-76a1e8ecfe01" />
106106

107107

108-
[View example report](https://prod.cytetype.nygen.io/report/e70e2883-7713-4121-94f2-5b57eabd1468?v=260303)
108+
[View example report](https://cytetype.nygen.io/report/e70e2883-7713-4121-94f2-5b57eabd1468?v=260303)
109109

110110
---
111111

cytetype/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
__version__ = "0.19.3"
1+
__version__ = "0.19.4"
22

33
import requests
44

cytetype/api/client.py

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -375,21 +375,25 @@ def fetch_job_results(
375375
def _sleep_with_spinner(
376376
seconds: int,
377377
progress: ProgressDisplay | None,
378-
cluster_status: dict[str, str],
378+
job_status: str,
379379
) -> None:
380380
"""Sleep for specified seconds while updating spinner animation.
381381
382382
Args:
383383
seconds: Number of seconds to sleep
384384
progress: ProgressDisplay instance (if showing progress)
385-
cluster_status: Current cluster status for display
385+
job_status: Current overall job status for display
386386
"""
387387
for _ in range(seconds * 2):
388388
if progress:
389-
progress.update(cluster_status)
389+
progress.update(job_status)
390390
time.sleep(0.5)
391391

392392

393+
def _log_report_cta(report_url: str) -> None:
394+
logger.info(f"\n[TRACK PROGRESS]\n{report_url}")
395+
396+
393397
def wait_for_completion(
394398
base_url: str,
395399
auth_token: str | None,
@@ -401,26 +405,26 @@ def wait_for_completion(
401405
"""Poll job until completion and return results."""
402406
progress = ProgressDisplay() if show_progress else None
403407
start_time = time.time()
408+
report_url = f"{base_url.rstrip('/')}/report/{job_id}"
404409

405-
logger.info(f"CyteType job (id: {job_id}) submitted. Polling for results...")
406-
407-
# Initial delay
408-
time.sleep(5)
409-
410-
# Show report URL
411-
report_url = f"{base_url}/report/{job_id}"
412-
logger.info(f"Report (updates automatically) available at: {report_url}")
410+
logger.info("CyteType job submitted.")
413411
logger.info(
414-
"If network disconnects, the results can still be fetched:\n"
412+
"If your session disconnects, results can still be fetched later with:\n"
415413
"`results = annotator.get_results()`"
416414
)
415+
_log_report_cta(report_url)
416+
417+
# Initial delay
418+
time.sleep(5)
417419

418420
consecutive_not_found = 0
421+
job_status = "pending"
422+
cluster_status: dict[str, str] = {}
419423

420424
while (time.time() - start_time) < timeout:
421425
try:
422426
status_data = get_job_status(base_url, auth_token, job_id)
423-
job_status = status_data.get("jobStatus")
427+
job_status = str(status_data.get("jobStatus") or "")
424428
cluster_status = status_data.get("clusterStatus", {})
425429

426430
# Reset 404 counter on valid response
@@ -429,20 +433,21 @@ def wait_for_completion(
429433

430434
if job_status == "completed":
431435
if progress:
432-
progress.finalize(cluster_status)
436+
progress.finalize("completed", cluster_status)
433437
logger.success(f"Job {job_id} completed successfully.")
434438
return fetch_job_results(base_url, auth_token, job_id)
435439

436440
elif job_status == "failed":
437441
if progress:
438-
progress.finalize(cluster_status)
442+
progress.finalize("failed", cluster_status)
443+
logger.info(f"Report:\n{report_url}")
439444
raise JobFailedError(f"Job {job_id} failed")
440445

441446
elif job_status in ["processing", "pending"]:
442447
logger.debug(
443448
f"Job {job_id} status: {job_status}. Waiting {poll_interval}s..."
444449
)
445-
_sleep_with_spinner(poll_interval, progress, cluster_status)
450+
_sleep_with_spinner(poll_interval, progress, job_status)
446451

447452
elif job_status == "not_found":
448453
consecutive_not_found += 1
@@ -459,24 +464,25 @@ def wait_for_completion(
459464
f"Status endpoint not ready for job {job_id}. "
460465
f"Waiting {poll_interval}s..."
461466
)
462-
_sleep_with_spinner(poll_interval, progress, cluster_status)
467+
_sleep_with_spinner(poll_interval, progress, job_status)
463468

464469
else:
465470
logger.warning(f"Unknown job status: '{job_status}'. Continuing...")
466-
_sleep_with_spinner(poll_interval, progress, cluster_status)
471+
_sleep_with_spinner(poll_interval, progress, job_status)
467472

468473
except APIError:
469474
# Let API errors (auth, etc.) bubble up immediately
470475
if progress:
471-
progress.finalize({})
476+
progress.finalize()
472477
raise
473478
except Exception as e:
474479
# Network errors - log and retry
475480
logger.debug(f"Error during polling: {e}. Retrying...")
476481
retry_interval = min(poll_interval, 5)
477-
_sleep_with_spinner(retry_interval, progress, cluster_status)
482+
_sleep_with_spinner(retry_interval, progress, job_status)
478483

479484
# Timeout reached
480485
if progress:
481-
progress.finalize({})
486+
progress.finalize("timed_out")
487+
logger.info(f"Report:\n{report_url}")
482488
raise TimeoutError(f"Job {job_id} did not complete within {timeout}s")

cytetype/api/progress.py

Lines changed: 138 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,116 +1,166 @@
11
import sys
2+
import time
3+
from html import escape
4+
from typing import Any, Callable, TextIO, cast
5+
6+
7+
def _in_notebook() -> bool:
8+
try:
9+
from IPython import get_ipython
10+
except ImportError:
11+
return False
12+
13+
shell_getter = cast(Callable[[], Any | None], get_ipython)
14+
shell = shell_getter()
15+
return bool(shell and shell.__class__.__name__ == "ZMQInteractiveShell")
16+
17+
18+
def _create_notebook_display_handle(message: str) -> Any | None:
19+
try:
20+
from IPython.display import display
21+
except ImportError:
22+
return None
23+
24+
_display: Callable[..., Any] = display
25+
return _display(_render_notebook_message(message), display_id=True)
26+
27+
28+
def _render_notebook_message(message: str) -> Any:
29+
try:
30+
from IPython.display import HTML
31+
except ImportError:
32+
return message
33+
34+
html_cls = cast(Callable[[str], Any], HTML)
35+
return html_cls(
36+
"<pre style='margin: 0; white-space: pre-wrap; font-family: monospace;'>"
37+
f"{escape(message)}"
38+
"</pre>"
39+
)
240

341

442
class ProgressDisplay:
5-
"""Manages terminal progress display during job polling."""
6-
7-
# Class constants
8-
COLORS = {
9-
"completed": "\033[92m",
10-
"processing": "\033[93m",
11-
"pending": "\033[94m",
12-
"failed": "\033[91m",
13-
"reset": "\033[0m",
14-
}
15-
SYMBOLS = {"completed": "✓", "processing": "⟳", "pending": "○", "failed": "✗"}
43+
"""Manages terminal and notebook progress display during job polling."""
44+
45+
COLORS = {"failed": "\033[91m", "reset": "\033[0m"}
1646
SPINNER_CHARS = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
1747

18-
def __init__(self) -> None:
48+
def __init__(self, stream: TextIO | None = None) -> None:
49+
self.stream = stream or sys.stdout
50+
self._interactive = bool(
51+
hasattr(self.stream, "isatty") and self.stream.isatty()
52+
)
53+
self._use_notebook_display = not self._interactive and _in_notebook()
54+
self._display_handle: Any | None = None
55+
self._finalized = False
56+
self._last_plain_status: str | None = None
57+
self._start_time = time.monotonic()
1958
self.spinner_frame = 0
20-
self.last_status: dict[str, str] = {}
2159

22-
def update(self, cluster_status: dict[str, str]) -> None:
23-
"""Update progress display with current cluster status."""
24-
if not cluster_status:
60+
def update(self, job_status: str) -> None:
61+
"""Update progress display with the overall job status."""
62+
if self._finalized:
2563
return
2664

27-
# Always render to keep spinner animating
28-
self._render(cluster_status, is_final=False)
29-
30-
# Track last status for potential future use
31-
if cluster_status != self.last_status:
32-
self.last_status = cluster_status.copy()
65+
if self._interactive:
66+
message = self._build_running_line(job_status)
67+
print(f"\r{message}\033[K", end="", file=self.stream, flush=True)
68+
elif self._use_notebook_display:
69+
self._update_notebook_display(self._build_running_line(job_status))
70+
else:
71+
message = self._build_plain_line(job_status)
72+
if message != self._last_plain_status:
73+
print(message, file=self.stream, flush=True)
74+
self._last_plain_status = message
3375

34-
# Always increment spinner to show activity
3576
self.spinner_frame += 1
3677

37-
def finalize(self, cluster_status: dict[str, str]) -> None:
78+
def finalize(
79+
self,
80+
final_status: str | None = None,
81+
cluster_status: dict[str, str] | None = None,
82+
) -> None:
3883
"""Show final status and cleanup."""
39-
if cluster_status:
40-
self._render(cluster_status, is_final=True)
41-
print() # Ensure newline
42-
43-
def _render(self, cluster_status: dict[str, str], is_final: bool) -> None:
44-
"""Render status to terminal."""
45-
status_counts = self._count_statuses(cluster_status)
46-
progress_bar = self._build_progress_bar(cluster_status)
47-
status_line = self._build_status_line(
48-
progress_bar, status_counts, is_final=is_final
49-
)
84+
if self._finalized:
85+
return
86+
self._finalized = True
5087

51-
# Print status line
52-
if is_final:
53-
print(f"\r{status_line}{self.COLORS['reset']}")
54-
sys.stdout.flush()
55-
self._show_failed_clusters(cluster_status, status_counts["failed"])
56-
else:
57-
print(f"\r{status_line}{self.COLORS['reset']}", end="", flush=True)
58-
59-
def _count_statuses(self, cluster_status: dict[str, str]) -> dict[str, int]:
60-
"""Count occurrences of each status."""
61-
counts = {"completed": 0, "failed": 0}
62-
for status in cluster_status.values():
63-
counts[status] = counts.get(status, 0) + 1
64-
return counts
65-
66-
def _build_progress_bar(self, cluster_status: dict[str, str]) -> str:
67-
"""Build colored progress bar from cluster statuses."""
68-
progress_units = []
69-
for cluster_id in self._sorted_cluster_ids(cluster_status):
70-
status = cluster_status[cluster_id]
71-
color = self.COLORS.get(status, self.COLORS["reset"])
72-
symbol = self.SYMBOLS.get(status, "?")
73-
progress_units.append(f"{color}{symbol}{self.COLORS['reset']}")
74-
return "".join(progress_units)
75-
76-
def _build_status_line(
77-
self, progress_bar: str, counts: dict[str, int], is_final: bool
78-
) -> str:
79-
"""Build status line with progress bar and counts."""
80-
total = sum(counts.values())
81-
completed = counts["completed"]
82-
83-
if is_final:
84-
status_line = f"[DONE] [{progress_bar}] {completed}/{total}"
85-
if counts["failed"] > 0:
86-
status_line += f" ({counts['failed']} failed)"
87-
elif completed == total:
88-
status_line += " completed"
88+
if final_status is None:
89+
if self._interactive:
90+
print(file=self.stream, flush=True)
91+
return
92+
93+
message = self._build_final_line(final_status)
94+
if self._interactive:
95+
print(f"\r{message}\033[K", file=self.stream, flush=True)
96+
elif self._use_notebook_display:
97+
self._update_notebook_display(message)
8998
else:
90-
spinner = self.SPINNER_CHARS[self.spinner_frame % len(self.SPINNER_CHARS)]
91-
status_line = f"{spinner} [{progress_bar}] {completed}/{total} completed"
99+
print(message, file=self.stream, flush=True)
100+
101+
if final_status == "failed" and cluster_status:
102+
self._show_failed_clusters(cluster_status)
103+
104+
def _update_notebook_display(self, message: str) -> None:
105+
"""Update a single notebook output cell instead of printing many lines."""
106+
if self._display_handle is None:
107+
self._display_handle = _create_notebook_display_handle(message)
108+
if self._display_handle is None:
109+
self._use_notebook_display = False
110+
print(message, file=self.stream, flush=True)
111+
return
92112

93-
return status_line
113+
self._display_handle.update(_render_notebook_message(message))
94114

95-
def _show_failed_clusters(
96-
self, cluster_status: dict[str, str], failed_count: int
97-
) -> None:
98-
"""Show details of failed clusters."""
99-
if failed_count == 0:
100-
return
115+
def _build_running_line(self, job_status: str) -> str:
116+
spinner = self.SPINNER_CHARS[self.spinner_frame % len(self.SPINNER_CHARS)]
117+
elapsed = self._format_elapsed()
118+
return f"{spinner} {self._status_message(job_status)} {elapsed} elapsed"
119+
120+
def _build_plain_line(self, job_status: str) -> str:
121+
return self._status_message(job_status)
122+
123+
@staticmethod
124+
def _build_final_line(final_status: str) -> str:
125+
if final_status == "completed":
126+
return "[DONE] CyteType job completed."
127+
if final_status == "failed":
128+
return "[FAILED] CyteType job failed."
129+
if final_status == "timed_out":
130+
return "[TIMEOUT] CyteType job timed out."
131+
return "[STOPPED] CyteType job stopped."
101132

133+
@staticmethod
134+
def _status_message(job_status: str) -> str:
135+
if job_status == "pending":
136+
return "CyteType job queued..."
137+
if job_status == "processing":
138+
return "CyteType job running..."
139+
if job_status == "not_found":
140+
return "Waiting for CyteType job to start..."
141+
return "Waiting for CyteType results..."
142+
143+
def _format_elapsed(self) -> str:
144+
elapsed = int(time.monotonic() - self._start_time)
145+
minutes, seconds = divmod(elapsed, 60)
146+
return f"{minutes:02d}:{seconds:02d}"
147+
148+
def _show_failed_clusters(self, cluster_status: dict[str, str]) -> None:
149+
"""Show details of failed clusters."""
102150
failed_details = []
103151
for cluster_id in self._sorted_cluster_ids(cluster_status):
104-
if cluster_status[cluster_id] == "failed":
105-
color = self.COLORS["failed"]
106-
symbol = self.SYMBOLS["failed"]
152+
if cluster_status[cluster_id] != "failed":
153+
continue
154+
155+
if self._interactive:
107156
failed_details.append(
108-
f"{color}{symbol} Cluster {cluster_id}{self.COLORS['reset']}"
157+
f"{self.COLORS['failed']} Cluster {cluster_id}{self.COLORS['reset']}"
109158
)
159+
else:
160+
failed_details.append(f"✗ Cluster {cluster_id}")
110161

111-
# Group into lines of 4
112162
for i in range(0, len(failed_details), 4):
113-
print(f" {' | '.join(failed_details[i : i + 4])}")
163+
print(f" {' | '.join(failed_details[i : i + 4])}", file=self.stream)
114164

115165
@staticmethod
116166
def _sorted_cluster_ids(cluster_status: dict[str, str]) -> list[str]:

0 commit comments

Comments
 (0)