Enhance GitHub repository synchronization: add support for modified repos, improve error handling, and implement live verification status

Author: MatALass

src/portfolio_auditor/collectors/github/collector.py

@@ -168,11 +168,15 @@ def _list_owner_repos(self, owner: str) -> tuple[list[dict[str, Any]], str]:
                 # Authenticated owner — /user/repos returns private repos too.
                 payloads = self.client.list_authenticated_user_repos()
             else:
-                # Unknown owner: try org first, fall back to public user listing.
-                try:
-                    self.client.get_org(owner)
-                    payloads = self.client.list_org_repos(owner)
-                except GitHubApiError:
+                # Unknown owner: try org first when the client supports org probing,
+                # otherwise fall back directly to the public user listing.
+                if hasattr(self.client, "get_org") and hasattr(self.client, "list_org_repos"):
+                    try:
+                        self.client.get_org(owner)
+                        payloads = self.client.list_org_repos(owner)
+                    except GitHubApiError:
+                        payloads = self.client.list_user_repos(owner)
+                else:
                     payloads = self.client.list_user_repos(owner)
 
             return payloads, "github_api"

src/portfolio_auditor/dashboard/app.py

@@ -3,7 +3,6 @@
 import os
 from datetime import datetime, timezone
 
-
 import streamlit as st
 
 from portfolio_auditor.dashboard.components.optimizer_view import render_optimizer_view
@@ -129,7 +128,7 @@ def _render_staleness_indicator(base_dir_mtime: float | None) -> None:
     if base_dir_mtime is None:
         return
 
-    age_seconds = (datetime.now(timezone.utc).timestamp() - base_dir_mtime)
+    age_seconds = datetime.now(timezone.utc).timestamp() - base_dir_mtime
     age_hours = age_seconds / 3600
 
     if age_hours < 1:
@@ -147,9 +146,10 @@ def _render_staleness_indicator(base_dir_mtime: float | None) -> None:
         unsafe_allow_html=True,
     )
     if age_hours >= 24:
-        st.warning("Artifacts are more than 24 h old. Run a fresh audit to pick up new repositories.", icon="⚠️")
-
-
+        st.warning(
+            "Artifacts are more than 24 h old. Run a fresh audit to pick up new repositories.",
+            icon="⚠️",
+        )
 
 
 def _format_relative_timestamp(moment: datetime | None) -> str:
@@ -172,6 +172,11 @@ def _render_repo_sync_status(sync_result: RepoSyncResult) -> None:
     delta = sync_result.delta
 
     st.markdown("### GitHub sync status")
+    if sync_result.verified_live:
+        st.success("Live verification: GitHub API reachable", icon="✅")
+    else:
+        st.warning("Live verification: unavailable, fallback-only view", icon="⚠️")
+
     st.caption(
         f"Checked {_format_relative_timestamp(delta.checked_at)} · Source: {sync_result.source.replace('_', ' ')}"
     )
@@ -183,23 +188,30 @@ def _render_repo_sync_status(sync_result: RepoSyncResult) -> None:
     if sync_result.warning:
         st.info(sync_result.warning)
 
-    if decision.should_refresh:
-        st.warning(decision.reason, icon="⚠️")
-    else:
-        st.success(decision.reason)
-
     if delta.latest_live_push_at or delta.latest_cached_push_at:
         st.caption(
             "Latest live push: "
             f"{_format_relative_timestamp(delta.latest_live_push_at)} · "
             "Latest cached push: "
             f"{_format_relative_timestamp(delta.latest_cached_push_at)}"
         )
+    if delta.latest_processed_audit_at:
+        st.caption(
+            f"Latest processed audit: {_format_relative_timestamp(delta.latest_processed_audit_at)}"
+        )
+
+    if decision.should_refresh:
+        st.warning(decision.reason, icon="⚠️")
+    elif sync_result.verified_live:
+        st.success(decision.reason)
+    else:
+        st.info(decision.reason)
 
     sections = [
         ("New repositories", delta.new_repos),
         ("Removed repositories", delta.removed_repos),
-        ("Changed repositories", delta.changed_repos),
+        ("Modified after processed audit", delta.modified_since_processed),
+        ("Changed since cached raw snapshot", delta.changed_repos),
     ]
     for title, repo_names in sections:
         if not repo_names:
@@ -228,16 +240,13 @@ def main() -> None:
     owners = discover_owners()
     default_owner = _resolve_default_owner()
 
-    # Build owner options: always include the resolved default so the sidebar
-    # is not empty on a fresh Streamlit Cloud deploy (no processed/ on disk yet).
     if owners:
         owner_options = owners
         if default_owner and default_owner not in owner_options:
             owner_options = [default_owner] + owner_options
     else:
         owner_options = [default_owner] if default_owner else ["MatALass"]
 
-    # Pre-select the default owner when possible.
     default_index = 0
     if default_owner and default_owner in owner_options:
         default_index = owner_options.index(default_owner)
@@ -250,7 +259,7 @@ def main() -> None:
             "Use refresh when repositories changed recently or when you want the latest "
             "local scan evidence."
         )
-        if st.button("Run fresh audit now", type="primary", use_container_width=True):
+        if st.button("Run fresh audit now", type="primary", width='stretch'):
             with st.spinner(
                 "Refreshing portfolio artifacts from GitHub. This can take a while for "
                 "larger portfolios."
@@ -281,17 +290,17 @@ def main() -> None:
     else:
         sync_error = None
 
-    # Staleness indicator — use mtime of ranking.json as the audit timestamp proxy.
     ranking_path = data.base_dir / "ranking.json"
     mtime = ranking_path.stat().st_mtime if ranking_path.exists() else None
     with st.sidebar:
         _render_staleness_indicator(mtime)
         st.markdown("---")
         if sync_result is not None:
             _render_repo_sync_status(sync_result)
-            if should_refresh_audit(sync_result).should_refresh and st.button(
+            refresh_decision = should_refresh_audit(sync_result)
+            if refresh_decision.should_refresh and st.button(
                 "Refresh audit to sync GitHub changes",
-                use_container_width=True,
+                width='stretch',
             ):
                 with st.spinner(
                     "Refreshing portfolio artifacts to include the latest GitHub changes."
@@ -384,4 +393,4 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    main()

src/portfolio_auditor/dashboard/repo_sync.py

@@ -3,7 +3,6 @@
 from dataclasses import dataclass
 from datetime import datetime, timezone
 
-from portfolio_auditor.audit_runner import AuditRunner
 from portfolio_auditor.collectors.github.client import GitHubApiError, GitHubClient
 from portfolio_auditor.collectors.github.collector import GitHubCollector
 from portfolio_auditor.models.repo_metadata import RepoMetadata
@@ -28,24 +27,38 @@ class RepoSyncDelta:
     new_repos: tuple[str, ...]
     removed_repos: tuple[str, ...]
     changed_repos: tuple[str, ...]
+    modified_since_processed: tuple[str, ...]
     latest_live_push_at: datetime | None
     latest_cached_push_at: datetime | None
+    latest_processed_audit_at: datetime | None
     checked_at: datetime
 
     @property
     def has_changes(self) -> bool:
-        return bool(self.new_repos or self.removed_repos or self.changed_repos)
+        return bool(
+            self.new_repos
+            or self.removed_repos
+            or self.changed_repos
+            or self.modified_since_processed
+        )
 
     @property
     def total_changed_count(self) -> int:
-        return len(self.new_repos) + len(self.removed_repos) + len(self.changed_repos)
+        repo_names = {
+            *self.new_repos,
+            *self.removed_repos,
+            *self.changed_repos,
+            *self.modified_since_processed,
+        }
+        return len(repo_names)
 
 
 @dataclass(frozen=True)
 class RepoSyncResult:
     owner: str
     delta: RepoSyncDelta
     source: str
+    verified_live: bool
     warning: str | None = None
 
 
@@ -74,31 +87,42 @@ def fetch_live_repo_sync_result(owner: str, settings: Settings) -> RepoSyncResul
     client = GitHubClient(settings)
     collector = GitHubCollector(client, settings)
     checked_at = datetime.now(timezone.utc)
+    latest_processed_at = latest_processed_audit_timestamp(owner, settings)
 
     try:
         payloads, payload_kind = collector._list_owner_repos(owner)
         if payload_kind == "normalized_snapshot":
             live_repos = [RepoMetadata.model_validate(item) for item in payloads]
+            live_repos = collector._apply_filters(live_repos)
             source = "cached_snapshot_fallback"
+            verified_live = False
             warning = (
-                "GitHub live sync fell back to the cached raw snapshot because the API could not be queried live."
+                "GitHub API was not queried live. The sync view is using the cached raw snapshot only, "
+                "so brand-new repositories cannot be confirmed until live access succeeds."
             )
         else:
             live_repos = [collector._parse_repo_payload(item) for item in payloads]
+            live_repos = collector._apply_filters(live_repos)
             source = "github_api"
+            verified_live = True
             warning = None
     except GitHubApiError as exc:
-        cached_repos = collector.load_raw_owner_snapshot(owner)
+        cached_repos = collector.load_raw_owner_snapshot(owner) if collector.has_raw_owner_snapshot(owner) else []
         delta = compare_repo_states(
             live_repos=cached_repos,
             cached_repos=cached_repos,
             checked_at=checked_at,
+            latest_processed_audit_at=latest_processed_at,
         )
         return RepoSyncResult(
             owner=owner,
             delta=delta,
             source="cached_snapshot_fallback",
-            warning=f"GitHub live sync unavailable: {exc}",
+            verified_live=False,
+            warning=(
+                "GitHub live sync unavailable. Using the cached raw snapshot only. "
+                f"Underlying error: {exc}"
+            ),
         )
     finally:
         client.close()
@@ -108,15 +132,23 @@ def fetch_live_repo_sync_result(owner: str, settings: Settings) -> RepoSyncResul
         live_repos=live_repos,
         cached_repos=cached_repos,
         checked_at=checked_at,
+        latest_processed_audit_at=latest_processed_at,
+    )
+    return RepoSyncResult(
+        owner=owner,
+        delta=delta,
+        source=source,
+        verified_live=verified_live,
+        warning=warning,
     )
-    return RepoSyncResult(owner=owner, delta=delta, source=source, warning=warning)
 
 
 def compare_repo_states(
     *,
     live_repos: list[RepoMetadata],
     cached_repos: list[RepoMetadata],
     checked_at: datetime,
+    latest_processed_audit_at: datetime | None = None,
 ) -> RepoSyncDelta:
     live_map = build_repo_snapshot_state_map(live_repos)
     cached_map = build_repo_snapshot_state_map(cached_repos)
@@ -137,27 +169,50 @@ def compare_repo_states(
         ):
             changed_repos.append(repo_name)
 
+    modified_since_processed = _compute_modified_since_processed(
+        live_map=live_map,
+        latest_processed_audit_at=latest_processed_audit_at,
+    )
+
     return RepoSyncDelta(
         live_repo_count=len(live_map),
         cached_repo_count=len(cached_map),
         new_repos=tuple(new_repos),
         removed_repos=tuple(removed_repos),
         changed_repos=tuple(changed_repos),
+        modified_since_processed=tuple(modified_since_processed),
         latest_live_push_at=_latest_push_at(live_map.values()),
         latest_cached_push_at=_latest_push_at(cached_map.values()),
+        latest_processed_audit_at=latest_processed_audit_at,
         checked_at=checked_at,
     )
 
 
 def should_refresh_audit(sync_result: RepoSyncResult) -> AuditDecision:
     delta = sync_result.delta
+    if not sync_result.verified_live:
+        return AuditDecision(
+            False,
+            "Live GitHub state could not be verified. Refresh decisions are disabled until API access succeeds.",
+        )
     if delta.new_repos:
         return AuditDecision(True, f"{len(delta.new_repos)} new repositories detected on GitHub.")
     if delta.removed_repos:
-        return AuditDecision(True, f"{len(delta.removed_repos)} repositories were removed from GitHub or are no longer visible.")
+        return AuditDecision(
+            True,
+            f"{len(delta.removed_repos)} repositories were removed from GitHub or are no longer visible.",
+        )
+    if delta.modified_since_processed:
+        return AuditDecision(
+            True,
+            f"{len(delta.modified_since_processed)} repositories were updated after the latest processed audit.",
+        )
     if delta.changed_repos:
-        return AuditDecision(True, f"{len(delta.changed_repos)} repositories changed since the last cached snapshot.")
-    return AuditDecision(False, "GitHub metadata matches the current cached snapshot.")
+        return AuditDecision(
+            True,
+            f"{len(delta.changed_repos)} repositories changed since the last cached raw snapshot.",
+        )
+    return AuditDecision(False, "Live GitHub metadata matches the current processed snapshot.")
 
 
 def latest_processed_audit_timestamp(owner: str, settings: Settings) -> datetime | None:
@@ -172,7 +227,24 @@ def latest_live_repo_push_timestamp(owner: str, settings: Settings) -> datetime
     return sync_result.delta.latest_live_push_at
 
 
-def _latest_push_at(items: list[RepoSnapshotState] | tuple[RepoSnapshotState, ...] | object) -> datetime | None:
+def _compute_modified_since_processed(
+    *,
+    live_map: dict[str, RepoSnapshotState],
+    latest_processed_audit_at: datetime | None,
+) -> list[str]:
+    if latest_processed_audit_at is None:
+        return []
+
+    modified_repo_names: list[str] = []
+    for repo_name in sorted(live_map):
+        repo = live_map[repo_name]
+        last_change = repo.pushed_at or repo.updated_at
+        if last_change > latest_processed_audit_at:
+            modified_repo_names.append(repo_name)
+    return modified_repo_names
+
+
+def _latest_push_at(items: object) -> datetime | None:
     timestamps: list[datetime] = []
     for item in items:
         if isinstance(item, RepoSnapshotState):