Fix issues #44, #34, #17, #43: OpenAI compat, Anthropic validation, keyring fallback, verbose logging

- #44: Use max_completion_tokens for newer OpenAI models (o1, o3, gpt-4o)
  that reject the deprecated max_tokens parameter
- #34: Detect Anthropic API URLs and use the anthropic SDK for connectivity
  tests instead of forcing OpenAI client on all providers
- #17: Add file-based fallback (credentials.json) when system keyring is
  unavailable (headless containers, RHEL). Support CODEWIKI_NO_KEYRING=1
  env var to force file-based storage
- #43: Add file-level and module-level verbose logging during dependency
  analysis, clustering, and doc generation phases

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: bdqnghi

codewiki/cli/adapters/doc_generator.py

@@ -186,9 +186,14 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
             components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph()
             self.job.statistics.total_files_analyzed = len(components)
             self.job.statistics.leaf_nodes = len(leaf_nodes)
-            
+
             if self.verbose:
-                self.progress_tracker.update_stage(1.0, f"Found {len(leaf_nodes)} leaf nodes")
+                self.progress_tracker.update_stage(0.8, f"Analyzed {len(components)} files, found {len(leaf_nodes)} leaf nodes")
+                # Log individual files analyzed
+                for comp_name in sorted(components.keys())[:20]:
+                    self.progress_tracker.update_stage(0.9, f"  File: {comp_name}")
+                if len(components) > 20:
+                    self.progress_tracker.update_stage(0.9, f"  ... and {len(components) - 20} more files")
         except Exception as e:
             raise APIError(f"Dependency analysis failed: {e}")
 
@@ -212,15 +217,22 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
         try:
             if os.path.exists(first_module_tree_path):
                 module_tree = file_manager.load_json(first_module_tree_path)
+                if self.verbose:
+                    self.progress_tracker.update_stage(0.5, "Loaded cached module tree")
             else:
+                if self.verbose:
+                    self.progress_tracker.update_stage(0.3, f"Clustering {len(leaf_nodes)} leaf nodes with LLM...")
                 module_tree = cluster_modules(leaf_nodes, components, backend_config)
                 file_manager.save_json(module_tree, first_module_tree_path)
-            
+
             file_manager.save_json(module_tree, module_tree_path)
             self.job.module_count = len(module_tree)
-            
+
             if self.verbose:
                 self.progress_tracker.update_stage(1.0, f"Created {len(module_tree)} modules")
+                for mod_name in sorted(module_tree.keys()):
+                    file_count = len(module_tree[mod_name]) if isinstance(module_tree[mod_name], list) else "?"
+                    self.progress_tracker.update_stage(1.0, f"  Module: {mod_name} ({file_count} files)")
         except Exception as e:
             raise APIError(f"Module clustering failed: {e}")
 
@@ -232,9 +244,12 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
             self.progress_tracker.update_stage(0.1, "Generating module documentation...")
 
         try:
+            if self.verbose:
+                self.progress_tracker.update_stage(0.2, f"Generating documentation for {self.job.module_count} modules...")
+
             # Run the actual documentation generation
             await doc_generator.generate_module_documentation(components, leaf_nodes)
-            
+
             if self.verbose:
                 self.progress_tracker.update_stage(0.9, "Creating repository overview...")
 

codewiki/cli/commands/config.py

@@ -490,13 +490,32 @@ def config_validate(quick: bool, verbose: bool):
 
         # Step 5: API connectivity test (unless --quick)
         if not quick:
+            if verbose:
+                click.echo()
+                click.echo("[5/5] Testing API connectivity...")
+                click.echo(f"      URL: {config.base_url}")
+
             try:
-                from openai import OpenAI
-                client = OpenAI(api_key=api_key, base_url=config.base_url)
-                response = client.models.list()
-                click.secho("✓ API connectivity test successful", fg="green")
+                base_url_lower = (config.base_url or "").lower()
+                if "api.anthropic.com" in base_url_lower:
+                    # Use Anthropic SDK for native Anthropic endpoints
+                    import anthropic
+                    client = anthropic.Anthropic(api_key=api_key)
+                    client.models.list(limit=1)
+                else:
+                    # Use OpenAI SDK for OpenAI-compatible endpoints
+                    from openai import OpenAI
+                    client = OpenAI(api_key=api_key, base_url=config.base_url)
+                    client.models.list()
+
+                if verbose:
+                    click.secho("      ✓ API responded successfully", fg="green")
+                else:
+                    click.secho("✓ API connectivity test successful", fg="green")
             except Exception as e:
                 click.secho("✗ API connectivity test failed", fg="red")
+                if verbose:
+                    click.echo(f"      Error: {e}")
                 sys.exit(EXIT_CONFIG_ERROR)
 
         # Success

codewiki/cli/config_manager.py

@@ -1,8 +1,14 @@
 """
 Configuration manager with keyring integration for secure credential storage.
+
+Supports fallback to file-based storage when system keyring is unavailable
+(e.g. headless containers, RHEL without Secret Service). Set the environment
+variable CODEWIKI_NO_KEYRING=1 to force file-based storage.
 """
 
 import json
+import os
+import logging
 from pathlib import Path
 from typing import Optional
 import keyring
@@ -12,6 +18,7 @@
 from codewiki.cli.utils.errors import ConfigurationError, FileSystemError
 from codewiki.cli.utils.fs import ensure_directory, safe_write, safe_read
 
+logger = logging.getLogger(__name__)
 
 # Keyring configuration
 KEYRING_SERVICE = "codewiki"
@@ -20,33 +27,63 @@
 # Configuration file location
 CONFIG_DIR = Path.home() / ".codewiki"
 CONFIG_FILE = CONFIG_DIR / "config.json"
+CREDENTIALS_FILE = CONFIG_DIR / "credentials.json"
 CONFIG_VERSION = "1.0"
 
 
 class ConfigManager:
     """
     Manages CodeWiki configuration with secure keyring storage for API keys.
-    
+
     Storage:
-        - API key: System keychain via keyring (macOS Keychain, Windows Credential Manager, 
+        - API key: System keychain via keyring (macOS Keychain, Windows Credential Manager,
                   Linux Secret Service)
+        - Fallback: ~/.codewiki/credentials.json when keyring is unavailable
         - Other settings: ~/.codewiki/config.json
+
+    Set CODEWIKI_NO_KEYRING=1 to skip keyring and use file-based storage.
     """
-    
+
     def __init__(self):
         """Initialize the configuration manager."""
         self._api_key: Optional[str] = None
         self._config: Optional[Configuration] = None
+        self._force_no_keyring = os.environ.get("CODEWIKI_NO_KEYRING", "").strip() in ("1", "true", "yes")
         self._keyring_available = self._check_keyring_available()
-    
+
     def _check_keyring_available(self) -> bool:
         """Check if system keyring is available."""
+        if self._force_no_keyring:
+            logger.debug("Keyring disabled via CODEWIKI_NO_KEYRING")
+            return False
         try:
             # Try to get/set a test value
             keyring.get_password(KEYRING_SERVICE, "__test__")
             return True
-        except KeyringError:
+        except (KeyringError, Exception):
             return False
+
+    def _load_api_key_from_file(self) -> Optional[str]:
+        """Load API key from fallback credentials file."""
+        if not CREDENTIALS_FILE.exists():
+            return None
+        try:
+            content = safe_read(CREDENTIALS_FILE)
+            data = json.loads(content)
+            return data.get("api_key")
+        except (json.JSONDecodeError, FileSystemError):
+            return None
+
+    def _save_api_key_to_file(self, api_key: str):
+        """Save API key to fallback credentials file (plaintext)."""
+        ensure_directory(CONFIG_DIR)
+        data = {"api_key": api_key}
+        safe_write(CREDENTIALS_FILE, json.dumps(data, indent=2))
+        # Restrict file permissions (owner read/write only)
+        try:
+            CREDENTIALS_FILE.chmod(0o600)
+        except OSError:
+            pass
 
     def load(self) -> bool:
         """
@@ -70,12 +107,14 @@ def load(self) -> bool:
 
             self._config = Configuration.from_dict(data)
 
-            # Load API key from keyring
-            try:
-                self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT)
-            except KeyringError:
-                # Keyring unavailable, API key will be None
-                pass
+            # Load API key from keyring, falling back to file
+            if self._keyring_available:
+                try:
+                    self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT)
+                except (KeyringError, Exception):
+                    pass
+            if self._api_key is None:
+                self._api_key = self._load_api_key_from_file()
 
             return True
         except (json.JSONDecodeError, FileSystemError) as e:
@@ -154,17 +193,23 @@ def save(
         if self._config.base_url and self._config.main_model and self._config.cluster_model:
             self._config.validate()
 
-        # Save API key to keyring
+        # Save API key to keyring, falling back to file
         if api_key is not None:
             self._api_key = api_key
-            try:
-                keyring.set_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT, api_key)
-            except KeyringError as e:
-                # Fallback: warn about keyring unavailability
-                raise ConfigurationError(
-                    f"System keychain unavailable: {e}\n"
-                    f"Please ensure your system keychain is properly configured."
-                )
+            if self._keyring_available:
+                try:
+                    keyring.set_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT, api_key)
+                except (KeyringError, Exception):
+                    # Keyring failed at runtime — fall back to file
+                    self._keyring_available = False
+                    self._save_api_key_to_file(api_key)
+                    logger.warning(
+                        "System keychain unavailable. API key stored in %s "
+                        "(plaintext). Set CODEWIKI_NO_KEYRING=1 to suppress this warning.",
+                        CREDENTIALS_FILE
+                    )
+            else:
+                self._save_api_key_to_file(api_key)
 
         # Save non-sensitive config to JSON
         config_data = {
@@ -179,17 +224,20 @@ def save(
 
     def get_api_key(self) -> Optional[str]:
         """
-        Get API key from keyring.
-        
+        Get API key from keyring or fallback file.
+
         Returns:
             API key or None if not set
         """
         if self._api_key is None:
-            try:
-                self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT)
-            except KeyringError:
-                pass
-        
+            if self._keyring_available:
+                try:
+                    self._api_key = keyring.get_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT)
+                except (KeyringError, Exception):
+                    pass
+            if self._api_key is None:
+                self._api_key = self._load_api_key_from_file()
+
         return self._api_key
 
     def get_config(self) -> Optional[Configuration]:
@@ -219,12 +267,19 @@ def is_configured(self) -> bool:
         return self._config.is_complete()
 
     def delete_api_key(self):
-        """Delete API key from keyring."""
-        try:
-            keyring.delete_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT)
-            self._api_key = None
-        except KeyringError:
-            pass
+        """Delete API key from keyring and fallback file."""
+        if self._keyring_available:
+            try:
+                keyring.delete_password(KEYRING_SERVICE, KEYRING_API_KEY_ACCOUNT)
+            except (KeyringError, Exception):
+                pass
+        # Also remove fallback credentials file
+        if CREDENTIALS_FILE.exists():
+            try:
+                CREDENTIALS_FILE.unlink()
+            except OSError:
+                pass
+        self._api_key = None
 
     def clear(self):
         """Clear all configuration (file and keyring)."""