Enhance search_servers documentation and update server entry structure

- Updated documentation to include repository information and separate MCP endpoints from source code repositories.
- Added `source_code_url` field to `ServerEntry` struct for better clarity on source code links.
- Modified tests to validate the new `source_code_url` field and ensure correct parsing of server entries.

Author: Dumbris

docs/search_servers.md

@@ -21,14 +21,22 @@ The search_servers functionality now includes an experimental repository detecti
 ```
 
 **Enhanced Output Format:**
-Results now include repository information when packages are detected:
+Results now include repository information when packages are detected and clearly separate MCP endpoints from source code repositories:
+
+**Field Descriptions:**
+- `url`: **MCP endpoint URL only** - Direct connection URL for remote MCP servers (e.g., `https://weather.example.com/mcp`)
+- `source_code_url`: **Source code repository URL** - Link to the source code repository (e.g., GitHub URLs)
+- `connectUrl`: Alternative connection URL for remote servers
+- `installCmd`: Command to install the server locally (npm/pip)
+
 ```json
 [
   {
     "id": "weather-service",
     "name": "Weather MCP Server",
     "description": "Provides weather data via MCP",
     "url": "https://weather.example.com/mcp",
+    "source_code_url": "https://github.com/example/weather-mcp-server",
     "installCmd": "npm install weather-mcp-server",
     "repository_info": {
       "npm": {
@@ -61,12 +69,13 @@ User Workflow:
     "id": "weather",
     "name": "WeatherInfo",
     "description": "Provides real-time weather data",
-    "url": "https://weather.mcp.run/mcp/", 
+    "url": "https://weather.mcp.run/mcp/",
+    "source_code_url": "https://github.com/mcprun/weather-service",
     "updatedAt": "2025-05-01T12:00:00Z"
   }
 ]
 
-In this example, the result indicates a server named “WeatherInfo”, with a base MCP endpoint URL. The user or agent can then call:
+In this example, the result indicates a server named “WeatherInfo”, with a base MCP endpoint URL for direct connection and a separate source code repository URL. The user or agent can then call:
 
 mcpproxy upstream_servers add --name "WeatherInfo" --url "https://weather.mcp.run/mcp/"
 
@@ -106,6 +115,7 @@ Example Output (for Example 1):
     "name": "WeatherInfo",
     "description": "Provides real-time weather data",
     "url": "https://weather.mcp.run/mcp/",
+    "source_code_url": "https://github.com/mcprun/weather-service",
     "updatedAt": "2025-05-01T12:00:00Z"
   },
   {
@@ -114,11 +124,12 @@ Example Output (for Example 1):
     "name": "ForecastPro",
     "description": "7-day weather forecast tool",
     "url": "https://forecast.mcp.run/mcp/",
+    "source_code_url": "https://github.com/mcprun/forecast-pro",
     "updatedAt": "2025-04-20T08:30:00Z"
   }
 ]
 
-In a real scenario, the above JSON could be printed to the console or returned as an MCP tool response. The user/agent sees two matching servers from MCP Run and can choose one to add. Each entry includes url (the base endpoint to connect to the MCP server) so that no additional lookup is required beyond this search.
+In a real scenario, the above JSON could be printed to the console or returned as an MCP tool response. The user/agent sees two matching servers from MCP Run and can choose one to add. Each entry includes url (the base endpoint to connect to the MCP server) and source_code_url (link to the source repository) so that no additional lookup is required beyond this search.
 
 ⚙️ Architecture & Flowchart
 

internal/oauth/config.go

@@ -6,9 +6,7 @@ import (
 	"mcpproxy-go/internal/config"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
-	"strings"
 	"sync"
 	"time"
 
@@ -90,15 +88,6 @@ func CreateOAuthConfig(serverConfig *config.ServerConfig) *client.OAuthConfig {
 
 	// Determine the correct OAuth server metadata URL based on the server URL
 	var authServerMetadataURL string
-	if serverConfig.URL != "" {
-		// Extract base URL from server URL and construct the well-known metadata endpoint
-		if u, err := parseBaseURL(serverConfig.URL); err == nil {
-			authServerMetadataURL = u + "/.well-known/oauth-authorization-server"
-			logger.Debug("Setting OAuth server metadata URL",
-				zap.String("server", serverConfig.Name),
-				zap.String("auth_server_metadata_url", authServerMetadataURL))
-		}
-	}
 
 	oauthConfig := &client.OAuthConfig{
 		ClientID:              "",                         // Will be obtained via Dynamic Client Registration
@@ -115,7 +104,7 @@ func CreateOAuthConfig(serverConfig *config.ServerConfig) *client.OAuthConfig {
 		zap.Strings("scopes", scopes),
 		zap.Bool("pkce_enabled", true),
 		zap.String("redirect_uri", callbackServer.RedirectURI),
-		zap.String("auth_server_metadata_url", authServerMetadataURL))
+		zap.String("discovery_mode", "automatic")) // Changed from explicit metadata URL to automatic discovery
 
 	return oauthConfig
 }
@@ -281,29 +270,6 @@ func GetGlobalCallbackManager() *CallbackServerManager {
 	return globalCallbackManager
 }
 
-// parseBaseURL extracts the base URL (scheme + host) from a full URL
-func parseBaseURL(fullURL string) (string, error) {
-	if fullURL == "" {
-		return "", fmt.Errorf("empty URL")
-	}
-
-	// Handle URLs that might not have a scheme
-	if !strings.HasPrefix(fullURL, "http://") && !strings.HasPrefix(fullURL, "https://") {
-		fullURL = "https://" + fullURL
-	}
-
-	u, err := url.Parse(fullURL)
-	if err != nil {
-		return "", fmt.Errorf("invalid URL: %w", err)
-	}
-
-	if u.Scheme == "" || u.Host == "" {
-		return "", fmt.Errorf("invalid URL: missing scheme or host")
-	}
-
-	return fmt.Sprintf("%s://%s", u.Scheme, u.Host), nil
-}
-
 // ShouldUseOAuth determines if OAuth should be attempted for a given server
 // Headers are tried first if configured, then OAuth as fallback on auth errors
 func ShouldUseOAuth(serverConfig *config.ServerConfig) bool {

internal/registries/new_parsers_test.go

@@ -57,8 +57,8 @@ func TestParseAzureMCPDemo(t *testing.T) {
 		if server.Name != "io.github.21st-dev/magic-mcp" {
 			t.Errorf("Expected name 'io.github.21st-dev/magic-mcp', got '%s'", server.Name)
 		}
-		if server.URL != "https://github.com/21st-dev/magic-mcp" {
-			t.Errorf("Expected URL 'https://github.com/21st-dev/magic-mcp', got '%s'", server.URL)
+		if server.SourceCodeURL != "https://github.com/21st-dev/magic-mcp" {
+			t.Errorf("Expected SourceCodeURL 'https://github.com/21st-dev/magic-mcp', got '%s'", server.SourceCodeURL)
 		}
 		if server.UpdatedAt != "2025-05-15T04:52:52Z" {
 			t.Errorf("Expected UpdatedAt '2025-05-15T04:52:52Z', got '%s'", server.UpdatedAt)

internal/registries/search.go

@@ -166,9 +166,9 @@ func applyBatchRepositoryGuessing(ctx context.Context, servers []ServerEntry, gu
 		server := &servers[i]
 		var githubURL string
 
-		// Check if server has a URL that looks like a GitHub repository
-		if server.URL != "" && isGitHubURL(server.URL) {
-			githubURL = server.URL
+		// Check if server has a SourceCodeURL that looks like a GitHub repository
+		if server.SourceCodeURL != "" && isGitHubURL(server.SourceCodeURL) {
+			githubURL = server.SourceCodeURL
 		}
 
 		// Add to batch if we found a GitHub URL
@@ -735,7 +735,7 @@ func parsePulseWithoutGuesser(rawData interface{}) []ServerEntry {
 
 		// Store source_code_url for later batch processing
 		if sourceCodeURL, ok := itemMap["source_code_url"].(string); ok && sourceCodeURL != "" {
-			server.URL = sourceCodeURL
+			server.SourceCodeURL = sourceCodeURL
 		}
 
 		servers = append(servers, server)
@@ -788,7 +788,7 @@ func parseAzureMCPDemoWithoutGuesser(rawData interface{}) []ServerEntry {
 		if repo, ok := itemMap["repository"].(map[string]interface{}); ok {
 			if repoURL, ok := repo["url"].(string); ok && repoURL != "" {
 				// Store repository URL for later batch processing
-				server.URL = repoURL
+				server.SourceCodeURL = repoURL
 			}
 		}
 

internal/registries/search_test.go

@@ -1085,22 +1085,22 @@ func TestApplyBatchRepositoryGuessing(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	// Create test servers with GitHub URLs
+	// Create test servers with GitHub URLs in SourceCodeURL field
 	servers := []ServerEntry{
 		{
-			ID:   "server1",
-			Name: "Test Server 1",
-			URL:  "https://github.com/facebook/react",
+			ID:            "server1",
+			Name:          "Test Server 1",
+			SourceCodeURL: "https://github.com/facebook/react",
 		},
 		{
-			ID:   "server2",
-			Name: "Test Server 2",
-			URL:  "https://github.com/nonexistent/package123",
+			ID:            "server2",
+			Name:          "Test Server 2",
+			SourceCodeURL: "https://github.com/nonexistent/package123",
 		},
 		{
 			ID:   "server3",
 			Name: "Test Server 3",
-			URL:  "https://example.com/not-github", // Non-GitHub URL
+			URL:  "https://example.com/not-github", // Non-GitHub URL (this is an MCP endpoint)
 		},
 		{
 			ID:   "server4",
@@ -1149,22 +1149,22 @@ func TestApplyBatchRepositoryGuessing_DuplicateURLs(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	// Create servers with duplicate GitHub URLs
+	// Create servers with duplicate GitHub URLs in SourceCodeURL field
 	servers := []ServerEntry{
 		{
-			ID:   "server1",
-			Name: "Test Server 1",
-			URL:  "https://github.com/facebook/react",
+			ID:            "server1",
+			Name:          "Test Server 1",
+			SourceCodeURL: "https://github.com/facebook/react",
 		},
 		{
-			ID:   "server2",
-			Name: "Test Server 2",
-			URL:  "https://github.com/lodash/lodash",
+			ID:            "server2",
+			Name:          "Test Server 2",
+			SourceCodeURL: "https://github.com/lodash/lodash",
 		},
 		{
-			ID:   "server3",
-			Name: "Test Server 3",
-			URL:  "https://github.com/facebook/react", // Duplicate URL
+			ID:            "server3",
+			Name:          "Test Server 3",
+			SourceCodeURL: "https://github.com/facebook/react", // Duplicate URL
 		},
 	}
 
@@ -1271,15 +1271,15 @@ func TestParsePulseWithoutGuesser(t *testing.T) {
 	assert.Equal(t, "Express MCP server for web development", servers[0].Description)
 	assert.Equal(t, "npx -y express-mcp-server", servers[0].InstallCmd)
 	assert.Equal(t, "https://express.example.com/mcp", servers[0].ConnectURL)
-	assert.Equal(t, "https://github.com/user/express-mcp-server", servers[0].URL)
+	assert.Equal(t, "https://github.com/user/express-mcp-server", servers[0].SourceCodeURL)
 
 	// Second server
 	assert.Equal(t, "weather-server", servers[1].ID)
 	assert.Equal(t, "weather-server", servers[1].Name)
 	assert.Equal(t, "Weather data MCP server", servers[1].Description)
 	assert.Equal(t, "", servers[1].InstallCmd) // No package info
 	assert.Equal(t, "", servers[1].ConnectURL)
-	assert.Equal(t, "https://github.com/user/weather-server", servers[1].URL)
+	assert.Equal(t, "https://github.com/user/weather-server", servers[1].SourceCodeURL)
 }
 
 func TestParseAzureMCPDemoWithoutGuesser(t *testing.T) {
@@ -1309,7 +1309,7 @@ func TestParseAzureMCPDemoWithoutGuesser(t *testing.T) {
 	assert.Equal(t, "azure-demo-1", server.ID)
 	assert.Equal(t, "Azure Demo Server", server.Name)
 	assert.Equal(t, "Demo server for Azure MCP (v1.0.0)", server.Description)
-	assert.Equal(t, "https://github.com/microsoft/azure-mcp-demo", server.URL)
+	assert.Equal(t, "https://github.com/microsoft/azure-mcp-demo", server.SourceCodeURL)
 	assert.Equal(t, "2024-01-01", server.UpdatedAt)
 }
 

internal/registries/types.go

@@ -16,15 +16,16 @@ type RegistryEntry struct {
 
 // ServerEntry represents an MCP server discovered via a registry
 type ServerEntry struct {
-	ID          string `json:"id"`
-	Name        string `json:"name"`
-	Description string `json:"description"`
-	URL         string `json:"url"`                  // MCP endpoint for direct connection
-	InstallCmd  string `json:"installCmd,omitempty"` // Command to install the server locally
-	ConnectURL  string `json:"connectUrl,omitempty"` // Alternative connection URL for remote servers
-	UpdatedAt   string `json:"updatedAt,omitempty"`
-	CreatedAt   string `json:"createdAt,omitempty"`
-	Registry    string `json:"registry,omitempty"` // Which registry this came from
+	ID            string `json:"id"`
+	Name          string `json:"name"`
+	Description   string `json:"description"`
+	URL           string `json:"url"`                       // MCP endpoint for remote server connections only
+	SourceCodeURL string `json:"source_code_url,omitempty"` // URL to source code repository
+	InstallCmd    string `json:"installCmd,omitempty"`      // Command to install the server locally
+	ConnectURL    string `json:"connectUrl,omitempty"`      // Alternative connection URL for remote servers
+	UpdatedAt     string `json:"updatedAt,omitempty"`
+	CreatedAt     string `json:"createdAt,omitempty"`
+	Registry      string `json:"registry,omitempty"` // Which registry this came from
 
 	// Repository detection information
 	RepositoryInfo *experiments.GuessResult `json:"repository_info,omitempty"` // Detected npm/pypi package info

internal/upstream/client.go

@@ -1217,15 +1217,28 @@ func (c *Client) performOAuthAuthorization(ctx context.Context, oauthHandler *mc
 		zap.String("server", c.config.Name),
 		zap.String("redirect_uri", callbackServer.RedirectURI))
 
-	// Step 1: Dynamic Client Registration
+	// Step 1: Dynamic Client Registration with panic recovery
 	c.logger.Info("Performing Dynamic Client Registration",
 		zap.String("server", c.config.Name))
 
-	if err := oauthHandler.RegisterClient(ctx, "mcpproxy-go"); err != nil {
+	var registrationErr error
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				c.logger.Error("Dynamic Client Registration panicked",
+					zap.String("server", c.config.Name),
+					zap.Any("panic", r))
+				registrationErr = fmt.Errorf("Dynamic Client Registration failed due to invalid OAuth server metadata: %v", r)
+			}
+		}()
+		registrationErr = oauthHandler.RegisterClient(ctx, "mcpproxy-go")
+	}()
+
+	if registrationErr != nil {
 		c.logger.Error("Dynamic Client Registration failed",
 			zap.String("server", c.config.Name),
-			zap.Error(err))
-		return fmt.Errorf("Dynamic Client Registration failed: %w", err)
+			zap.Error(registrationErr))
+		return fmt.Errorf("Dynamic Client Registration failed: %w", registrationErr)
 	}
 
 	c.logger.Info("Dynamic Client Registration successful",