- more logging formatting revisions

- fixed http connection error handling in dotnews pipeline
- updated daily script
- added a11y functions to web app
- added dark/light theme to web app

Author: cledantec

LICENSE.md

@@ -539,13 +539,13 @@ def print_config_summary(test_connections: bool = False):
     log("AI/LLM:")
     log(f"  Gemini Model: {GEMINI_MODEL}")
     log(f"  Gemini Embed Model: {GEMINI_EMBED_MODEL}")
-    log(f"  Gemini API Key: {'*' * 10}...{GEMINI_API_KEY[-4:] if GEMINI_API_KEY else 'NOT SET'}\n")
-    log(f"  Processing: LLM Max Workers = {LLM_MAX_WORKERS}")
+    log(f"  Gemini API Key: {'*' * 10}...{GEMINI_API_KEY[-4:] if GEMINI_API_KEY else 'NOT SET'}")
+    log(f"  LLM Max Workers: {LLM_MAX_WORKERS}\n")
     log("Google Services:")
     log(f"  Drive Folder ID: {GOOGLE_DRIVE_FOLDER_ID or 'NOT SET'}")
-    log(f"  Drive Credentials: {GOOGLE_CREDENTIALS_PATH}")
-    log(f"  Gmail Credentials: {GMAIL_CREDENTIALS_PATH}")
-    log(f"  Gmail Token: {GMAIL_TOKEN_PATH} {'✔' if GMAIL_TOKEN_PATH and GMAIL_TOKEN_PATH.exists() else '(not found)'}\n")
+    log(f"  Drive Credentials: {GOOGLE_CREDENTIALS_PATH} {'✔' if GOOGLE_CREDENTIALS_PATH and GOOGLE_CREDENTIALS_PATH.exists() else '✗ Not found'}")
+    log(f"  Gmail Credentials: {GMAIL_CREDENTIALS_PATH} {'✔' if GMAIL_CREDENTIALS_PATH and GMAIL_CREDENTIALS_PATH.exists() else '✗ Not found'}")
+    log(f"  Gmail Token: {GMAIL_TOKEN_PATH} {'✔' if GMAIL_TOKEN_PATH and GMAIL_TOKEN_PATH.exists() else '✗ Not found'}\n")
     log("Directories:")
     log(f"  Vector DB: {VECTORDB_DIR}")
     log(f"  Data Downloads: {DATA_DOWNLOAD_DIR}\n")

README.md

@@ -46,7 +46,7 @@ def extract_date_from_pdf(pdf_content: bytes) -> Optional[date]:
             if month:
                 return date(int(year_str), month, int(day_str))
     except Exception as e:
-        log_warning(f"Could not extract date from PDF: {e}")
+        log_warning(f"[DOTNEWS] Could not extract date from PDF: {e}")
     return None
 
 
@@ -86,7 +86,7 @@ def load(cls, path: Path) -> "SyncState":
                     # Old format: entire file is {renamed_filename: {...}}
                     # We can't recover original filenames, so start fresh
                     # but keep track of renamed files to avoid reprocessing
-                    log_info("Migrating legacy sync state format...")
+                    log_info(f"[DOTNEWS] Migrating legacy sync state format...")
                     return cls(downloaded_files={}, last_sync=data.get("last_sync"))
 
                 return cls(downloaded_files=data.get("downloaded_files", {}), last_sync=data.get("last_sync"))
@@ -147,14 +147,14 @@ def generate_renamed_filename(extracted_date: Optional[date], year: int, month:
 def list_pdfs_in_month(year: int, month: int) -> list[tuple[str, str]]:
     """List PDF files in a given month's WP uploads directory."""
     url = f"{DOTNEWS_BASE_URL}{DOTNEWS_WP_UPLOADS_PATH}/{year}/{month:02d}/"
-    log_success(f"Downloading from {url}...")
+    log_success(f"[DOTNEWS] Downloading from {url}...")
 
     try:
         response = requests.get(url, timeout=30)
         response.raise_for_status()
     except requests.RequestException as e:
-        log_error(f"Error fetching directory listing: {e}")
-        return []
+        log_error(f"[DOTNEWS] {e}")
+        raise
 
     pdfs = []
     matches = PDF_HREF_PATTERN.findall(response.text)
@@ -200,11 +200,11 @@ def download_pdf_content(url: str) -> Optional[bytes]:
         content = response.content
         if not content[:4].startswith(b"%PDF"):
             content_type = response.headers.get("Content-Type", "").lower()
-            log_error(f"Content is not a PDF (Content-Type: {content_type})")
+            log_error(f"[DOTNEWS] Content is not a PDF (Content-Type: {content_type})")
             return None
         return content
     except requests.RequestException as e:
-        log_error(f"Error downloading: {e}")
+        log_error(f"[DOTNEWS] Error downloading: {e}")
         return None
 
 
@@ -231,50 +231,50 @@ def download_pdfs(output_dir: Optional[Path] = None, start_year: Optional[int] =
     sync_state_path = config.DOTNEWS_SYNC_STATE_FILENAME
     sync_state = SyncState.load(sync_state_path)
 
-    log_debug(f"Output directory: {output_dir}")
-    log_debug(f"Date range: {start_year}-{start_month:02d} to {end_year}-{end_month:02d}")
-    log_debug(f"Previously downloaded: {len(sync_state.downloaded_files)} files")
+    log_debug(f"[DOTNEWS] Output directory: {output_dir}")
+    log_debug(f"[DOTNEWS] Date range: {start_year}-{start_month:02d} to {end_year}-{end_month:02d}")
+    log_debug(f"[DOTNEWS] Previously downloaded: {len(sync_state.downloaded_files)} files")
 
     downloaded_paths = []
     current = date(start_year, start_month, 1)
     end = date(end_year, end_month, 1)
 
     while current <= end:
         year, month = current.year, current.month
-        log_debug(f"Processing {year}-{month:02d}...")
+        log_debug(f"[DOTNEWS] Processing {year}-{month:02d}...")
 
         pdfs = list_pdfs_in_month(year, month)
         if not pdfs:
-            log_debug(f"  No PDFs found for {year}-{month:02d}")
+            log_debug(f"[DOTNEWS]   No PDFs found for {year}-{month:02d}")
         else:
-            log_debug(f"  Found {len(pdfs)} PDF(s)")
+            log_debug(f"[DOTNEWS]   Found {len(pdfs)} PDF(s)")
 
             for original_filename, pdf_url in pdfs:
                 # Check sync state BEFORE downloading - skip if we've seen this filename
                 if sync_state.is_downloaded(original_filename):
-                    log_debug(f"  Skipping {original_filename} (already downloaded)")
+                    log_debug(f"[DOTNEWS]   Skipping {original_filename} (already downloaded)")
                     continue
 
                 # Only download files we haven't seen before
-                log_debug(f"  Downloading {original_filename}...")
+                log_debug(f"[DOTNEWS]   Downloading {original_filename}...")
                 pdf_content = download_pdf_content(pdf_url)
                 if pdf_content is None:
                     continue
 
                 # Extract date from PDF content for human-readable filename
                 extracted_date = extract_date_from_pdf(pdf_content)
                 if extracted_date:
-                    log_debug(f"  Extracted date: {extracted_date.isoformat()}")
+                    log_debug(f"[DOTNEWS]   Extracted date: {extracted_date.isoformat()}")
                 else:
-                    log_debug("  Could not extract date, using fallback")
+                    log_debug(f"[DOTNEWS]   Could not extract date, using fallback")
 
                 renamed_filename = generate_renamed_filename(extracted_date, year, month)
                 output_path = output_dir / renamed_filename
 
                 with open(output_path, "wb") as f:
                     f.write(pdf_content)
 
-                log_debug(f"  Saved as: {output_path.name} ({len(pdf_content) / 1024:.1f} KB)")
+                log_debug(f"[DOTNEWS]   Saved as: {output_path.name} ({len(pdf_content) / 1024:.1f} KB)")
 
                 # Record original filename -> renamed filename mapping
                 sync_state.mark_downloaded(original_filename, renamed_filename)
@@ -304,21 +304,21 @@ def download_pdfs(output_dir: Optional[Path] = None, start_year: Optional[int] =
         try:
             start_year, start_month = map(int, args.start.split("-"))
         except ValueError:
-            log_error(f"Error: Invalid start date format '{args.start}'. Use YYYY-MM.")
+            log_error(f"[DOTNEWS] Error: Invalid start date format '{args.start}'. Use YYYY-MM.")
             sys.exit(1)
 
     end_year, end_month = None, None
     if args.end:
         try:
             end_year, end_month = map(int, args.end.split("-"))
         except ValueError:
-            log_error(f"Error: Invalid end date format '{args.end}'. Use YYYY-MM.")
+            log_error(f"[DOTNEWS] Error: Invalid end date format '{args.end}'. Use YYYY-MM.")
             sys.exit(1)
 
     output_dir = Path(args.output_dir) if args.output_dir else None
     results = download_pdfs(output_dir=output_dir, start_year=start_year, start_month=start_month, end_year=end_year, end_month=end_month)
 
     if results:
-        log_success(f"Downloaded {len(results)} new files.")
+        log_success(f"[DOTNEWS] Downloaded {len(results)} new files.")
     else:
-        log_success("No new files downloaded.")
+        log_success(f"[DOTNEWS] No new files downloaded.")

config.py

@@ -78,7 +78,7 @@ def get_gmail_credentials(interactive: bool = True) -> Credentials:
         try:
             creds = Credentials.from_authorized_user_file(str(token_path), SCOPES)
         except Exception as e:
-            log_error(f"Could not load existing token: {e}")
+            log_error(f"[GMAIL] Could not load existing token: {e}")
 
     # If valid credentials exist, return them
     if creds and creds.valid:
@@ -89,10 +89,10 @@ def get_gmail_credentials(interactive: bool = True) -> Credentials:
         try:
             creds.refresh(Request())
             token_path.write_text(creds.to_json())
-            log_debug("✔  Token refreshed successfully")
+            log_debug(f"[GMAIL] ✔  Token refreshed successfully")
             return creds
         except Exception as e:
-            log_error(f"Token refresh failed: {e}")
+            log_error(f"[GMAIL] Token refresh failed: {e}")
             creds = None
 
     # No valid credentials - need user interaction
@@ -109,10 +109,10 @@ def get_gmail_credentials(interactive: bool = True) -> Credentials:
         raise AuthenticationRequiredError(auth_url)
 
     # Interactive mode: open browser for user to authorize
-    log_info("Opening browser for Gmail authorization...")
+    log_info(f"[GMAIL] Opening browser for Gmail authorization...")
     creds = flow.run_local_server(port=8080, access_type="offline", prompt="consent")
     token_path.write_text(creds.to_json())
-    log_success("Authorization complete, token saved")
+    log_success(f"[GMAIL] Authorization complete, token saved")
     return creds
 
 
@@ -286,7 +286,7 @@ def extract_events_with_llm(text: str, source: str, publication_date: str = None
         return events
 
     except Exception as e:
-        log_error(f"Error extracting events with LLM: {e}")
+        log_error(f"[GMAIL] Error extracting events with LLM: {e}")
         return []
 
 
@@ -349,7 +349,7 @@ def insert_events_to_db(events: List[Dict]) -> int:
                     )
                     inserted_count += 1
                 except Exception as e:
-                    log_error(f"Could not insert event '{event.get('event_name')}': {e}")
+                    log_error(f"[GMAIL] Could not insert event '{event.get('event_name')}': {e}")
 
         conn.commit()
     finally:
@@ -374,7 +374,7 @@ def sync_email_newsletters_to_sql(interactive: bool = True) -> dict:
         # email_errors = [e for e in errors if "EMAIL" in e.upper()]
         # if email_errors:
         #     for error in email_errors:
-        #         log_debug(f"✗ Configuration error: {error}")
+        #         log_debug(f"[GMAIL] ✗ Configuration error: {error}")
         #         stats["errors"].append(error)
         #     return stats
 
@@ -383,28 +383,26 @@ def sync_email_newsletters_to_sql(interactive: bool = True) -> dict:
         processed_ids = state.get("processed_email_ids", [])
 
         # Connect to Gmail API
-        log_debug("Connecting to Gmail API...")
         try:
             service = get_gmail_service(interactive=interactive)
         except AuthenticationRequiredError as e:
             # Non-interactive mode and auth needed
             stats["auth_required"] = True
             stats["auth_url"] = e.auth_url
             error_msg = f"Gmail authentication required. Visit: {e.auth_url}"
-            log_error(f"{error_msg}")
+            log_error(f"[GMAIL] {error_msg}")
             stats["errors"].append(error_msg)
             return stats
 
-        log_success("Authenticated with Gmail.")
+        log_success(f"[GMAIL] Authenticated.")
 
         # Get recent newsletters
-        log_debug(f"Scanning inbox for newsletters from last {config.EMAIL_LOOKBACK_DAYS} days...")
+        log_debug(f"[GMAIL] Scanning inbox for newsletters from last {config.EMAIL_LOOKBACK_DAYS} days...")
         newsletters = get_recent_newsletters(service, processed_ids, days_back=config.EMAIL_LOOKBACK_DAYS)
 
-        log_info(f"Found {len(newsletters)} new newsletters to process.")
+        log_info(f"[GMAIL] Found {len(newsletters)} new newsletters to process.")
 
         if not newsletters:
-            log_info("No new newsletters. Exiting.")
             return stats
 
         all_events = []
@@ -422,60 +420,58 @@ def sync_email_newsletters_to_sql(interactive: bool = True) -> dict:
                 except Exception:
                     pub_date = datetime.now().strftime("%Y-%m-%d")
 
-                log_debug(f"[{i}/{len(newsletters)}] Processing: {subject[:60]}...")
+                log_debug(f"[GMAIL] [{i}/{len(newsletters)}] Processing: {subject[:60]}...")
 
                 email_text = extract_text_from_email(msg)
                 pdf_texts = extract_pdf_attachments(msg)
 
                 full_text = email_text
                 if pdf_texts:
                     full_text += "\n\n" + "\n\n".join(pdf_texts)
-                    log_debug(f"Found {len(pdf_texts)} PDF attachment(s)")
+                    log_debug(f"[GMAIL] Found {len(pdf_texts)} PDF attachment(s)")
 
                 if not full_text.strip():
-                    log_debug("No text content found")
+                    log_debug(f"[GMAIL] No text content found")
                     continue
 
                 events = extract_events_with_llm(full_text, source=f"Email: {subject}", publication_date=pub_date)
 
                 if events:
-                    log_debug(f"Extracted {len(events)} events")
+                    log_debug(f"[GMAIL] Extracted {len(events)} events")
                     all_events.extend(events)
                 else:
-                    log_debug("No events found")
+                    log_debug(f"[GMAIL] No events found")
 
                 processed_ids.append(email_id)
                 stats["emails_processed"] += 1
                 stats["events_extracted"] += len(events)
 
             except Exception as e:
                 error_msg = f"Error processing email {email_id}: {str(e)}"
-                log_error(f"{error_msg}")
+                log_error(f"[GMAIL] {error_msg}")
                 stats["errors"].append(error_msg)
 
         if all_events:
-            log_debug(f"Inserting {len(all_events)} events into database...")
+            log_debug(f"[GMAIL] Inserting {len(all_events)} events into database...")
             inserted = insert_events_to_db(all_events)
             stats["events_inserted"] = inserted
-            log_success(f"Inserted {inserted} events successfully")
+            log_success(f"[GMAIL] Inserted {inserted} events successfully")
 
         state["processed_email_ids"] = processed_ids[-1000:]
         save_email_sync_state(state)
-        log_success("Sync state saved.")
+        log_success(f"[GMAIL] Sync state saved.")
 
     except Exception as e:
         error_msg = f"Fatal error during sync: {str(e)}"
-        log_error(f"{error_msg}")
+        log_error(f"[GMAIL] {error_msg}")
         stats["errors"].append(error_msg)
 
-    log("\n" + "=" * 60)
-    log("   Gmail Sync Summary")
-    log("=" * 60)
-    log(f"     Emails processed: {stats['emails_processed']}")
-    log(f"     Events extracted: {stats['events_extracted']}")
-    log(f"     Events inserted (SQL): {stats['events_inserted']}")
-    log(f"     Errors: {len(stats['errors'])}")
-    log("=" * 60)
+    log("[GMAIL]   Sync Summary")
+    log("[GMAIL] " + "=" * 60)
+    log(f"[GMAIL]     Emails processed: {stats['emails_processed']}")
+    log(f"[GMAIL]     Events extracted: {stats['events_extracted']}")
+    log(f"[GMAIL]     Events inserted (SQL): {stats['events_inserted']}")
+    log(f"[GMAIL]     Errors: {len(stats['errors'])}")
 
     return stats
 
@@ -491,21 +487,21 @@ def sync_email_newsletters_to_sql(interactive: bool = True) -> dict:
     try:
         if args.auth:
             # Just do authentication
-            log_debug("Running Gmail OAuth authentication...")
+            log_debug(f"[GMAIL] Running Gmail OAuth authentication...")
             get_gmail_credentials(interactive=True)
-            log_success("Authentication successful! Token saved.")
+            log_success(f"[GMAIL] Authentication successful! Token saved.")
         else:
             # Run full sync
             interactive = not args.non_interactive
             sync_email_newsletters_to_sql(interactive=interactive)
     except KeyboardInterrupt:
-        log_warning("Interrupted by user. Exiting...")
+        log_warning(f"[GMAIL] Interrupted by user. Exiting...")
         sys.exit(1)
     except AuthenticationRequiredError as e:
-        log_warning(f"Authentication required!")
-        log_info(f"Visit this URL to authorize: {e.auth_url}")
-        log_info("\nOr run: python email_to_calendar_sql.py --auth")
+        log_warning(f"[GMAIL] Authentication required!")
+        log_info(f"[GMAIL] Visit this URL to authorize: {e.auth_url}")
+        log_info(f"[GMAIL] \nOr run: python email_to_calendar_sql.py --auth")
         sys.exit(2)
     except Exception as e:
-        log_error(f"Fatal error: {e}")
+        log_error(f"[GMAIL] Fatal error: {e}")
         sys.exit(1)