Merge pull request #38 from zhujian0805/main

feat: implement unified fetching framework and eliminate code duplication

Author: zhujian0805

REFACTOR_FETCH_DUPLICATION.md

@@ -4,15 +4,13 @@
 across different AI tool handlers.
 """
 
-import concurrent.futures
 import json
 import logging
-import shutil
-import threading
 from pathlib import Path
 from typing import Dict, List, Optional, Type
 
 from ..repo_loader import RepoConfigLoader
+from ..fetching.base import BaseEntityFetcher, RepoConfig
 from .base import BaseAgentHandler
 from .claude import ClaudeAgentHandler
 from .codebuddy import CodebuddyAgentHandler
@@ -116,6 +114,10 @@ def __init__(self, config_dir: Optional[Path] = None):
         self.repos_file = self.config_dir / "agent_repos.json"
         self.config_dir.mkdir(parents=True, exist_ok=True)
 
+        # Initialize fetcher with agent parser
+        from ..fetching.parsers import AgentParser
+        self.fetcher = BaseEntityFetcher(parser=AgentParser())
+
         # Initialize handlers
         self._handlers: Dict[str, BaseAgentHandler] = {}
         for app_name, handler_class in AGENT_HANDLERS.items():
@@ -311,120 +313,35 @@ def fetch_agents_from_repos(self, max_workers: int = 8) -> List[Agent]:
             self._init_default_repos_file()
             repos = self._load_repos()
 
-        # Filter enabled repos
-        enabled_repos = {
-            repo_id: repo for repo_id, repo in repos.items() if repo.enabled
-        }
-
-        if not enabled_repos:
-            logger.warning("No enabled repositories found")
-            return []
+        # Convert AgentRepo objects to RepoConfig objects for the fetcher
+        repo_configs = [
+            RepoConfig(
+                owner=repo.owner,
+                name=repo.name,
+                branch=repo.branch,
+                path=repo.agents_path,
+                enabled=repo.enabled
+            )
+            for repo in repos.values()
+        ]
 
-        logger.info(f"Fetching agents from {len(enabled_repos)} repositories in parallel")
+        # Fetch using unified fetcher
+        agents = self.fetcher.fetch_from_repos(
+            repos=repo_configs,
+            max_workers=max_workers
+        )
 
-        all_agents = []
+        # Update installed status from existing agents
         existing_agents = self._load_agents()
-
-        # Thread-safe storage for results
-        agents_results = []
-        lock = threading.Lock()
-
-        # Use claude handler for fetching (all repos use same format)
-        handler = self.get_handler("claude")
-
-        def process_repository(repo_id: str, repo: AgentRepo):
-            """Process a single repository to extract agents."""
-            try:
-                agents = self._fetch_agents_from_repo(repo, handler)
-                for agent in agents:
-                    if agent.key in existing_agents:
-                        agent.installed = existing_agents[agent.key].installed
-
-                with lock:
-                    agents_results.extend(agents)
-
-                logger.info(f"Found {len(agents)} agents in {repo_id}")
-                return len(agents)
-            except Exception as e:
-                logger.warning(f"Failed to fetch agents from {repo_id}: {e}")
-                return 0
-
-        # Use ThreadPoolExecutor for parallel processing
-        actual_workers = min(max_workers, len(enabled_repos))
-        logger.debug(f"Using {actual_workers} concurrent workers")
-
-        with concurrent.futures.ThreadPoolExecutor(max_workers=actual_workers) as executor:
-            # Submit all tasks
-            future_to_repo = {
-                executor.submit(process_repository, repo_id, repo): repo_id
-                for repo_id, repo in enabled_repos.items()
-            }
-
-            # Wait for all tasks to complete
-            for future in concurrent.futures.as_completed(future_to_repo):
-                repo_id = future_to_repo[future]
-                try:
-                    agent_count = future.result()
-                    logger.debug(f"Completed processing {repo_id}: {agent_count} agents")
-                except Exception as e:
-                    logger.error(f"Exception processing {repo_id}: {e}")
-
-        all_agents = agents_results
-
-        # Merge and save
-        for agent in all_agents:
+        for agent in agents:
+            if agent.key in existing_agents:
+                agent.installed = existing_agents[agent.key].installed
             existing_agents[agent.key] = agent
-        self._save_agents(existing_agents)
-
-        logger.info(f"Total agents fetched: {len(all_agents)}")
-        return all_agents
-
-    def _fetch_agents_from_repo(
-        self, repo: AgentRepo, handler: BaseAgentHandler
-    ) -> List[Agent]:
-        """Fetch agents from a single repository.
-
-        Args:
-            repo: The repository to fetch from
-            handler: The handler to use for parsing
 
-        Returns:
-            List of agents found
-        """
-        # Use the new Fetcher class similar to awesome-claude-agents
-        fetcher = Fetcher()
-
-        repo_data = {
-            "owner": repo.owner,
-            "name": repo.name,
-            "branch": repo.branch,
-            "agentsPath": repo.agents_path or "agents"
-        }
-
-        agents_data = fetcher.fetch_agents_from_repo(repo_data)
-        agents = []
-
-        for agent_data in agents_data:
-            # Convert to Agent model
-            filename = agent_data["file_path"].split("/")[-1] if "/" in agent_data["file_path"] else agent_data["file_path"]
-
-            agent = Agent(
-                key=f"{repo.owner}/{repo.name}:{agent_data['name']}",
-                name=agent_data["name"],
-                description=agent_data["description"],
-                filename=filename,
-                installed=False,
-                repo_owner=repo.owner,
-                repo_name=repo.name,
-                repo_branch=repo.branch,
-                agents_path=repo.agents_path,
-                readme_url=f"https://github.com/{repo.owner}/{repo.name}/blob/{repo.branch}/{agent_data['file_path']}",
-                tools=agent_data.get("source_data", {}).get("tools", []),
-                color=agent_data.get("source_data", {}).get("color"),
-            )
-            agents.append(agent)
-            logger.debug(f"Found agent: {agent.key}")
+        # Save updated agents
+        self._save_agents(existing_agents)
 
+        logger.info(f"Total agents fetched: {len(agents)}")
         return agents
 
     def fetch_agents_from_external_sources(self) -> List[Agent]:

code_assistant_manager/agents/manager.py

@@ -390,12 +390,19 @@ def install_plugin(
         marketplace = _resolve_plugin_conflict(plugin, app)
 
     # Use @ syntax for Claude CLI compatibility, but show : syntax in output
-    plugin_ref = f"{plugin}@{marketplace}" if marketplace else plugin
+    # For Claude CLI, extract just the repo name from owner/repo format
+    claude_marketplace = marketplace
+    if getattr(handler, "uses_cli_plugin_commands", False) and marketplace and "/" in marketplace:
+        # Claude CLI uses just repo name, not owner/repo
+        claude_marketplace = marketplace.split("/")[-1]  # Extract "claude-plugins-official" from "anthropics/claude-plugins-official"
+        plugin_ref = f"{plugin}@{claude_marketplace}"
+    else:
+        plugin_ref = f"{plugin}@{marketplace}" if marketplace else plugin
     display_ref = f"{marketplace}:{plugin}" if marketplace else plugin
     typer.echo(f"{Colors.CYAN}Installing plugin: {display_ref}...{Colors.RESET}")
 
     if getattr(handler, "uses_cli_plugin_commands", False):
-        success, msg = handler.install_plugin(plugin, marketplace)
+        success, msg = handler.install_plugin(plugin, claude_marketplace)
     else:
         # Install directly from CAM-configured marketplace (no app CLI required)
         from code_assistant_manager.plugins import PluginManager

code_assistant_manager/cli/plugins/plugin_install_commands.py

@@ -224,6 +224,8 @@ def _show_available_plugins(manager: PluginManager, query: Optional[str] = None,
     """Show available plugins from all configured marketplaces."""
     from code_assistant_manager.plugins.fetch import fetch_repo_info
     from code_assistant_manager.cli.plugins.plugin_discovery_commands import _filter_plugins, _display_plugin
+    import concurrent.futures
+    import threading
 
     all_repos = manager.get_all_repos()
     if not all_repos:
@@ -234,16 +236,50 @@ def _show_available_plugins(manager: PluginManager, query: Optional[str] = None,
     all_plugins = []
     repo_sources = {}  # Track which repo each plugin comes from
 
-    for repo_name, repo in all_repos.items():
-        if not repo.repo_owner or not repo.repo_name:
-            continue
+    # Thread-safe storage for results
+    results_lock = threading.Lock()
+    fetch_results = []
 
-        # Fetch repo info
-        info = fetch_repo_info(
-            repo.repo_owner, repo.repo_name, repo.repo_branch or "main"
-        )
-        if not info:
-            continue
+    def fetch_single_repo(repo_name: str, repo):
+        """Fetch plugins from a single repository."""
+        try:
+            # Fetch repo info
+            info = fetch_repo_info(
+                repo.repo_owner, repo.repo_name, repo.repo_branch or "main"
+            )
+            if not info:
+                return None
+
+            result_data = {"repo_name": repo_name, "repo": repo, "info": info}
+            return result_data
+        except Exception as e:
+            logger.warning(f"Failed to fetch from {repo_name}: {e}")
+            return None
+
+    # Use ThreadPoolExecutor for parallel fetching (5-10x speedup!)
+    actual_workers = min(8, len(all_repos))  # Max 8 concurrent requests
+    logger.debug(f"Fetching from {len(all_repos)} repositories with {actual_workers} workers")
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=actual_workers) as executor:
+        # Submit all fetch tasks
+        future_to_repo = {
+            executor.submit(fetch_single_repo, repo_name, repo): repo_name
+            for repo_name, repo in all_repos.items()
+            if repo.repo_owner and repo.repo_name
+        }
+
+        # Collect results as they complete
+        for future in concurrent.futures.as_completed(future_to_repo):
+            result = future.result()
+            if result:
+                with results_lock:
+                    fetch_results.append(result)
+
+    # Process the results
+    for result_data in fetch_results:
+        repo_name = result_data["repo_name"]
+        repo = result_data["repo"]
+        info = result_data["info"]
 
         if info.type == "marketplace":
             # Add plugins from marketplace with their source

code_assistant_manager/cli/plugins/plugin_management_commands.py

@@ -0,0 +1,15 @@
+"""Unified fetching framework for skills, agents, and plugins."""
+
+from .base import BaseEntityFetcher, EntityParser, RepoConfig
+from .parallel import ParallelFetcher
+from .repository import GitRepository
+from .cache import FetchCache
+
+__all__ = [
+    "BaseEntityFetcher",
+    "EntityParser",
+    "RepoConfig",
+    "ParallelFetcher",
+    "GitRepository",
+    "FetchCache",
+]