Add server instructions and make all instructions externalized and overridable (#161)

Author: keyurva

packages/datacommons-mcp/.env.sample

@@ -43,6 +43,20 @@ DC_TYPE=base
 # Default: "base_and_custom" for custom DC
 # DC_SEARCH_SCOPE=base_and_custom
 
+# =============================================================================
+# LESS COMMONLY USED OPTIONAL CONFIGURATION
+# =============================================================================
+
+# Path to directory containing markdown file overrides for server instructions and/or tool descriptions.
+# Supports partial overrides: only create files for the specific instructions or tools you want to replace. 
+# The system will fall back to package defaults for any file not found here.
+#
+# Expected structure inside this directory:
+# - server.md
+# - tools/{tool_name}.md
+# DC_INSTRUCTIONS_DIR=/path/to/custom/instructions
+
+
 # =============================================================================
 # NON-PROD ROOTS (optional, base DC only)
 # =============================================================================

packages/datacommons-mcp/datacommons_mcp/app.py

@@ -0,0 +1,120 @@
+# Copyright 2026 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Core application module for the DC MCP server.
+"""
+
+import json
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from fastmcp import FastMCP
+from fastmcp.tools.tool import Tool
+from pydantic import ValidationError
+
+from datacommons_mcp import settings
+from datacommons_mcp.clients import create_dc_client
+from datacommons_mcp.utils import read_external_content, read_package_content
+from datacommons_mcp.version import __version__
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+MCP_SERVER_NAME = "DC MCP Server"
+DEFAULT_INSTRUCTIONS_PACKAGE = "datacommons_mcp.instructions"
+SERVER_INSTRUCTION_FILE = "server.md"
+
+
+class DCApp:
+    """Core application wrapper for Data Commons MCP."""
+
+    def __init__(self) -> None:
+        """Initialize the application."""
+        # Load settings
+        try:
+            self.settings = settings.get_dc_settings()
+            settings_dict = self.settings.model_dump()
+            settings_dict["api_key"] = (
+                "<SET>" if settings_dict.get("api_key") else "<NOT_SET>"
+            )
+            logger.info("Loaded DC settings:\n%s", json.dumps(settings_dict, indent=2))
+        except ValidationError as e:
+            logger.error("Settings error: %s", e)
+            raise
+
+        # Create client
+        try:
+            self.client = create_dc_client(self.settings)
+        except Exception as e:
+            logger.error("Failed to create DC client: %s", e)
+            raise
+
+        # Load Server Instructions
+        server_instructions = self._load_instruction(SERVER_INSTRUCTION_FILE)
+
+        self.mcp = FastMCP(
+            MCP_SERVER_NAME,
+            version=__version__,
+            instructions=server_instructions,
+        )
+
+    def _load_instruction(self, filename: str) -> str:
+        """
+        Loads markdown content.
+        Priority:
+        1. DC_INSTRUCTIONS_DIR/{filename} (if set and exists)
+        2. Package default: datacommons_mcp/instructions/{filename}
+        """
+        # Check specific override
+        if self.settings.instructions_dir:
+            content = read_external_content(self.settings.instructions_dir, filename)
+            if content is not None:
+                logger.info(
+                    "Loaded custom instruction for %s from %s",
+                    filename,
+                    self.settings.instructions_dir,
+                )
+                return content
+            logger.debug(
+                "Custom instruction file %s not found in %s, falling back to default.",
+                filename,
+                self.settings.instructions_dir,
+            )
+
+        # Fallback to package resources
+        return read_package_content(DEFAULT_INSTRUCTIONS_PACKAGE, filename)
+
+    def register_tool(self, func: Callable[..., Any], instruction_file: str) -> None:
+        """Register a tool with instructions loaded from a file.
+
+        Args:
+            func: The tool function to register.
+            instruction_file: Path to instruction file relative to instructions dir.
+        """
+        description = self._load_instruction(instruction_file)
+        if not description:
+            logger.warning(
+                "No description found for tool %s from file %s",
+                func.__name__,
+                instruction_file,
+            )
+
+        # Create tool from function and add description
+        tool = Tool.from_function(func, description=description)
+        self.mcp.add_tool(tool)
+
+
+# Create global app instance
+app = DCApp()

packages/datacommons-mcp/datacommons_mcp/data_models/settings.py

@@ -53,6 +53,12 @@ class DCSettings(BaseSettings):
         description="Whether to use the legacy search-indicators endpoint (True) or the client library (False) for fetching indicators.",
     )
 
+    instructions_dir: str | None = Field(
+        default=None,
+        alias="DC_INSTRUCTIONS_DIR",
+        description="Directory containing custom instruction files (markdown overrides)",
+    )
+
 
 class BaseDCSettings(DCSettings):
     """Settings for base Data Commons instance."""

packages/datacommons-mcp/datacommons_mcp/instructions/__init__.py

@@ -0,0 +1 @@
+Act as a Data Commons Research Assistant. This server provides direct access to a massive, unified knowledge graph of aggregated statistical data from authoritative regional and global sources like the UN, World Bank, and Census Bureau. Use it to transform natural language queries into precise statistical insights by identifying specific indicators and retrieving their observations. It contains historical and recent data points on topics like demographics, economics, health, and environment across various geographic levels. It does not contain information on topics like real-time news, subjective viewpoints, or private corporate data. Crucially, every data point retrieved must be attributed to its original source provided in the tool output; never present statistics as "known facts" without citing the specific organization or dataset they originated from. Prioritize data integrity and transparency, ensuring that users understand both the metric and the provenance of the information provided.

packages/datacommons-mcp/datacommons_mcp/instructions/server.md

@@ -0,0 +1,66 @@
+Fetches observations for a statistical variable from Data Commons.
+
+**CRITICAL: Always validate variable-place combinations first**
+- You **MUST** call `search_indicators` first to verify that the variable exists for the specified place
+- Only use DCIDs returned by `search_indicators` - never guess or assume variable-place combinations
+- This ensures data availability and prevents errors from invalid combinations
+
+This tool can operate in two primary modes:
+1.  **Single Place Mode**: Get data for one specific place (e.g., "Population of California").
+2.  **Child Places Mode**: Get data for all child places of a certain type within a parent place (e.g., "Population of all counties in California").
+
+### Core Logic & Rules
+
+* **Variable Selection**: You **must** provide the `variable_dcid`.
+    * Variable DCIDs are unique identifiers for statistical variables in Data Commons and are returned by prior calls to the
+    `search_indicators` tool.
+
+* **Place Selection**: You **must** provide the `place_dcid`.
+    * **Important Note for Bilateral Data**: When fetching data for bilateral variables (e.g., exports from one country to another),
+    the `variable_dcid` often encodes one of the places (e.g., `TradeExports_FRA` refers to exports *to* France).
+    In such cases, the `place_dcid` parameter in `get_observations` should specify the *other* place involved in the bilateral relationship
+    (e.g., the exporter country, such as 'USA' for exports *from* USA).
+    The `search_indicators` tool's `places_with_data` field can help identify which place is the appropriate observation source for `place_dcid`.
+
+* **Mode Selection**:
+    * To get data for the specified place (e.g., California), **do not** provide `child_place_type`.
+    * To get data for all its children (e.g., all counties in California), you **must also** provide the `child_place_type` (e.g., "County").
+      **CRITICAL:** Before calling `get_observations` with `child_place_type`, you **MUST** first call `search_indicators` with child sampling to determine the correct child place type.
+      **Child Type Determination Logic:**
+      1. Use the `dcid_place_type_mappings` field from the `search_indicators` response to examine the types of sampled child places
+      2. Use the type that is common to ALL sampled child places
+      3. If more than one type is common to all child places, use the most specific type
+      4. If there is no common type across all sampled child places, use the majority type (50%+ threshold) if there's a clear majority
+      5. If there is no common type and no clear majority, this tool cannot be called with child-place mode - fall back to single-place mode `get_observations` calls for each place
+      **Note:** If you used child sampling in `search_indicators` to validate variable existence, you should still get data for ALL children of that type, not just the sampled subset.
+
+* **Data Volume Constraint**: When using **Child Places Mode** (when `child_place_type` is set), you **must** be conservative with your date range to avoid requesting too much data.
+    * Avoid requesting `'all'` data via the `date` parameter.
+    * **Instead, you must either request the `'latest'` data or provide a specific, bounded date range.**
+
+* **Date Filtering**: The tool filters observations by date using the following priority:
+    1.  **`date`**: The `date` parameter is required and can be one of the enum values 'all', 'latest', 'range', or a date string in the format 'YYYY', 'YYYY-MM', or 'YYYY-MM-DD'.
+    2.  **Date Range**: If `date` is set to 'range', you must specify a date range using `date_range_start` and/or `date_range_end`.
+        * If only `date_range_start` is specified, then the response will contain all observations starting at and after that date (inclusive).
+        * If only `date_range_end` is specified, then the response will contain all observations before and up to that date (inclusive).
+        * If both are specified, the response contains observations within the provided range (inclusive).
+        * Dates must be in `YYYY`, `YYYY-MM`, or `YYYY-MM-DD` format.
+    3.  **Default Behavior**: If you do not provide **any** date parameters (`date`, `date_range_start`, or `date_range_end`), the tool will automatically fetch only the `'latest'` observation.
+
+Args:
+  variable_dcid (str, required): The unique identifier (DCID) of the statistical variable.
+  place_dcid (str, required): The DCID of the place.
+  child_place_type (str, optional): The type of child places to get data for. **Use this to switch to Child Places Mode.**
+  source_override (str, optional): An optional source ID to force the use of a specific data source.
+  date (str, optional): An optional date filter. Accepts 'all', 'latest', 'range', or single date values of the format 'YYYY', 'YYYY-MM', or 'YYYY-MM-DD'. Defaults to 'latest' if no date parameters are provided.
+  date_range_start (str, optional): The start date for a range (inclusive). **Used only if `date` is set to'range'.**
+  date_range_end (str, optional): The end date for a range (inclusive). **Used only if `date` is set to'range'.**
+
+Returns:
+    The fetched observation data including:
+    - `variable`: Details about the statistical variable requested.
+    - `place_observations`: A list of observations, one entry per place. Each entry contains:
+        - `place`: Details about the observed place (DCID, name, type).
+        - `time_series`: A list of `(date, value)` tuples, where `date` is a string (e.g., "2022-01-01") and `value` is a float.
+    - `source_metadata`: Information about the primary data source used.
+    - `alternative_sources`: Details about other available data sources.