diff --git a/.gitignore b/.gitignore index 7f6619314..ef75016f2 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,6 @@ tests/test_data/rapl/* credentials* .codecarbon.config* scripts/agent-vm.personal.config.sh + +# Added by ggshield +.cache_ggshield diff --git a/carbonserver/tests/api/test_telemetry_schema_drift.py b/carbonserver/tests/api/test_telemetry_schema_drift.py index abc20d764..388e39963 100644 --- a/carbonserver/tests/api/test_telemetry_schema_drift.py +++ b/carbonserver/tests/api/test_telemetry_schema_drift.py @@ -7,7 +7,9 @@ from carbonserver.api.schemas_telemetry import TelemetryCreate as ServerTelemetryCreate REPO_ROOT = Path(__file__).resolve().parents[3] -CORE_TELEMETRY_SCHEMA_PATH = REPO_ROOT / "codecarbon" / "core" / "telemetry_schemas.py" +CORE_TELEMETRY_SCHEMA_PATH = ( + REPO_ROOT / "codecarbon" / "core" / "telemetry" / "schemas.py" +) def _load_core_telemetry_create(): diff --git a/codecarbon/cli/main.py b/codecarbon/cli/main.py index fd6545a3f..c5c741255 100644 --- a/codecarbon/cli/main.py +++ b/codecarbon/cli/main.py @@ -22,6 +22,7 @@ overwrite_local_config, ) from codecarbon.cli.monitor import run_and_monitor +from codecarbon.cli.telemetry_cli import normalize_telemetry_level, telemetry_app from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate, ProjectCreate from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker @@ -32,6 +33,7 @@ DEFAULT_ORGANIzATION_ID = "e60afa92-17b7-4720-91a0-1ae91e409ba1" codecarbon = typer.Typer(no_args_is_help=True) +codecarbon.add_typer(telemetry_app, name="telemetry") def main(): @@ -342,6 +344,15 @@ def monitor( str, typer.Option(help="Region/province for offline mode"), ] = None, + telemetry_level: Annotated[ + Optional[str], + typer.Option( + help=( + "Override telemetry tier for this run only " + "(disabled, minimal, or extensive)." + ), + ), + ] = None, ): """Monitor your machine's carbon emissions.""" @@ -350,6 +361,8 @@ def monitor( "measure_power_secs": measure_power_secs, "api_call_interval": api_call_interval, } + if telemetry_level is not None: + tracker_args["telemetry_level"] = normalize_telemetry_level(telemetry_level) # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode if offline: if not country_iso_code: diff --git a/codecarbon/cli/telemetry_cli.py b/codecarbon/cli/telemetry_cli.py new file mode 100644 index 000000000..279ab1e63 --- /dev/null +++ b/codecarbon/cli/telemetry_cli.py @@ -0,0 +1,252 @@ +"""CLI commands to configure CodeCarbon product telemetry tiers.""" + +from pathlib import Path +from typing import Optional + +import questionary +import typer +from rich import print +from typing_extensions import Annotated + +from codecarbon.cli.cli_utils import ( + create_new_config_file, + get_config, + overwrite_local_config, +) +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config +from codecarbon.core.telemetry import ( + DEFAULT_TELEMETRY_LEVEL, + TelemetryLevel, + TelemetrySettings, + parse_telemetry_level, +) + +telemetry_app = typer.Typer( + help="Configure product telemetry (disabled, minimal, or extensive).", + no_args_is_help=False, +) + +TIER_DESCRIPTIONS = { + "disabled": "No product telemetry.", + "minimal": "Environment and hardware only (private POST /telemetry).", + "extensive": "Minimal fields plus run metrics and public run summary.", +} + + +def normalize_telemetry_level(level: str) -> str: + """Validate and normalize a telemetry tier string for CLI use. + + Args: + level: User-provided tier name. + + Returns: + Canonical tier value. + + Raises: + typer.BadParameter: If the level is not a valid ``TelemetryLevel``. + """ + try: + return parse_telemetry_level(level).value + except ValueError as error: + raise typer.BadParameter(str(error)) from error + + +def resolve_config_path(config: Optional[Path], *, create: bool = False) -> Path: + """Resolve which config file to read or write. + + Args: + config: Explicit path from ``--config``, if any. + create: When True and no file exists, create ``./.codecarbon.config``. + + Returns: + Resolved config file path. + """ + if config is not None: + path = config.expanduser().resolve() + if create and not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + return path + local_path = Path.cwd().resolve() / ".codecarbon.config" + if local_path.exists(): + return local_path + global_path = (Path.home() / ".codecarbon.config").expanduser().resolve() + if global_path.exists(): + return global_path + if create: + local_path.write_text("[codecarbon]\n", encoding="utf-8") + return local_path + return local_path + + +def pick_config_path_interactive() -> Path: + """Prompt for which config file to update. + + Returns: + Path chosen by the user. + """ + home = Path.home() + global_path = (home / ".codecarbon.config").expanduser().resolve() + local_path = Path.cwd().resolve() / ".codecarbon.config" + options = [] + if global_path.exists(): + options.append(str(global_path)) + if local_path.exists() and local_path not in options: + options.append(str(local_path)) + options.append("Create new config file") + if not options: + options = ["Create new config file"] + choice = questionary.select( + "Which configuration file should store telemetry_level?", + choices=options, + ).ask() + if choice == "Create new config file": + return create_new_config_file() + return Path(choice).expanduser().resolve() + + +def write_telemetry_level(path: Path, level: str) -> None: + """Persist ``telemetry_level`` to a config file. + + Args: + path: Target ``.codecarbon.config`` path. + level: Validated tier value. + """ + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("[codecarbon]\n", encoding="utf-8") + overwrite_local_config("telemetry_level", level, path=path) + + +def print_telemetry_status(config_path: Optional[Path] = None) -> None: + """Print resolved telemetry settings. + + Without ``config_path``, uses the same merged file settings and env overlay + as ``EmissionsTracker``. With ``config_path``, inspects that file only. + + Args: + config_path: Optional single config file to inspect. + """ + if config_path is not None: + path = config_path.expanduser().resolve() + if not path.exists(): + print(f"[yellow]Config file not found:[/yellow] {path}") + print(f"Default tier: {DEFAULT_TELEMETRY_LEVEL.value} (not explicit)") + return + file_settings = get_config(path) + external_conf: dict[str, str] = {} + source_label = str(path) + else: + file_settings = get_config_file_settings() + external_conf = get_hierarchical_config() + source_label = "merged ~/.codecarbon.config + ./.codecarbon.config" + + settings = TelemetrySettings.resolve( + config_file_conf=file_settings, + external_conf=external_conf or None, + ) + level = settings.level + explicit = settings.is_explicit + stored = file_settings.get("telemetry_level") + print(f"Config source: {source_label}") + print(f"telemetry_level in file(s): {stored!r}") + print(f"Resolved tier: {level.value}") + print(f"Explicitly configured: {explicit}") + if not explicit: + print( + "[yellow]Minimal telemetry will be sent on each tracker stop " + "until you set telemetry_level.[/yellow]" + ) + + +def run_telemetry_interactive(config: Optional[Path] = None) -> None: + """Run the interactive telemetry configuration wizard. + + Args: + config: Optional fixed config path; otherwise prompt for file choice. + """ + print("CodeCarbon product telemetry") + print( + "Separate from your dashboard experiment (codecarbon config). " + "Controls optional usage analytics and public leaderboard data.\n" + ) + path = resolve_config_path(config) if config else None + if path is None or (config is None and not path.exists()): + path = pick_config_path_interactive() + else: + path = resolve_config_path(config, create=True) + + choices = [ + questionary.Choice("disabled — " + TIER_DESCRIPTIONS["disabled"], "disabled"), + questionary.Choice("minimal — " + TIER_DESCRIPTIONS["minimal"], "minimal"), + questionary.Choice( + "extensive — " + TIER_DESCRIPTIONS["extensive"], "extensive" + ), + ] + try: + current = get_config(path).get("telemetry_level") + except FileNotFoundError: + current = None + valid_levels = {member.value for member in TelemetryLevel} + default = current if current in valid_levels else "minimal" + level = questionary.select( + "Select telemetry_level:", + choices=choices, + default=default, + ).ask() + if level is None: + raise typer.Exit(0) + level = normalize_telemetry_level(level) + write_telemetry_level(path, level) + print(f"[green]Saved[/green] telemetry_level = {level} in {path}") + + +@telemetry_app.callback(invoke_without_command=True) +def telemetry_entry( + ctx: typer.Context, + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (default: local then global).", + ), + ] = None, +) -> None: + """Configure telemetry interactively when no subcommand is given.""" + if ctx.invoked_subcommand is None: + run_telemetry_interactive(config=config) + + +@telemetry_app.command("status") +def status( + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Inspect one file only; default matches EmissionsTracker merge.", + ), + ] = None, +) -> None: + """Print resolved telemetry tier and configuration source.""" + print_telemetry_status(config_path=config) + + +@telemetry_app.command("set") +def set_level( + level: Annotated[ + str, + typer.Argument(help="Telemetry tier: disabled, minimal, or extensive."), + ], + config: Annotated[ + Optional[Path], + typer.Option( + "--config", + help="Path to .codecarbon.config (creates ./.codecarbon.config if missing).", + ), + ] = None, +) -> None: + """Write telemetry_level to a config file.""" + path = resolve_config_path(config, create=True) + normalized = normalize_telemetry_level(level) + write_telemetry_level(path, normalized) + print(f"[green]Saved[/green] telemetry_level = {normalized} in {path}") diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index 9349b9dc1..a2746f973 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -30,6 +30,13 @@ def get_datetime_with_timezone(): return timestamp +def _round_coordinate(value, decimals: int = 1) -> float: + """Round a geographic coordinate for API payloads, treating None as zero.""" + if value is None: + return round(0.0, decimals) + return round(float(value), decimals) + + class ApiClient: # (AsyncClient) """ This class call the Code Carbon API @@ -267,9 +274,8 @@ def _create_run(self, experiment_id: str): cpu_model=self.conf.get("cpu_model"), gpu_count=self.conf.get("gpu_count"), gpu_model=self.conf.get("gpu_model"), - # Reduce precision for Privacy - longitude=round(self.conf.get("longitude", 0), 1), - latitude=round(self.conf.get("latitude", 0), 1), + longitude=_round_coordinate(self.conf.get("longitude")), + latitude=_round_coordinate(self.conf.get("latitude")), region=self.conf.get("region"), provider=self.conf.get("provider"), ram_total_size=self.conf.get("ram_total_size"), diff --git a/codecarbon/core/config.py b/codecarbon/core/config.py index 7cacea41d..65330f7a0 100644 --- a/codecarbon/core/config.py +++ b/codecarbon/core/config.py @@ -110,6 +110,31 @@ def normalize_gpu_ids( return None +def _config_file_paths() -> tuple[str, str]: + """Return resolved paths for global and local CodeCarbon config files.""" + cwd = Path.cwd() + home = Path.home() + global_path = str((home / ".codecarbon.config").expanduser().resolve()) + local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + return global_path, local_path + + +def get_config_file_settings() -> dict[str, str]: + """Return the ``[codecarbon]`` section from config files without environment overlay. + + Reads ``~/.codecarbon.config`` then ``./.codecarbon.config`` (local overrides global). + + Returns: + Configuration dict from files only. Empty when no file or section exists. + """ + config = configparser.ConfigParser() + global_path, local_path = _config_file_paths() + config.read([global_path, local_path]) + if "codecarbon" not in config: + return {} + return dict(config["codecarbon"]) + + def get_hierarchical_config(): """ Get the user-defined codecarbon configuration ConfigParser dictionnary @@ -137,13 +162,7 @@ def get_hierarchical_config(): dict: The final configuration dict parsed from global, local and environment configurations. **All values are strings**. """ - - config = configparser.ConfigParser() - - cwd = Path.cwd() - home = Path.home() - global_path = str((home / ".codecarbon.config").expanduser().resolve()) - local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) + global_path, local_path = _config_file_paths() if Path(global_path).exists(): logger.info( f"Codecarbon is taking the configuration from global file: {global_path}" @@ -155,7 +174,6 @@ def get_hierarchical_config(): f"Codecarbon is taking the configuration from the local file {local_path}" ) - config.read([global_path, local_path]) - config.read_dict(parse_env_config()) - - return dict(config["codecarbon"]) + conf = get_config_file_settings() + conf.update(parse_env_config().get("codecarbon", {})) + return conf diff --git a/codecarbon/core/telemetry/__init__.py b/codecarbon/core/telemetry/__init__.py new file mode 100644 index 000000000..efa31dbaa --- /dev/null +++ b/codecarbon/core/telemetry/__init__.py @@ -0,0 +1,39 @@ +"""Product telemetry sent at tracker stop (Tier 1 / Tier 2).""" + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.dispatcher import Telemetry +from codecarbon.core.telemetry.schemas import ( + MINIMAL_TELEMETRY_FIELDS, + TelemetryCreate, + TelemetryLevel, +) +from codecarbon.core.telemetry.settings import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + DEFAULT_TELEMETRY_LEVEL, + TELEMETRY_LEVEL_CONFIG_KEY, + TelemetryLevelSource, + TelemetrySettings, + parse_telemetry_level, +) + +__all__ = [ + "DEFAULT_TELEMETRY_API_KEY", + "DEFAULT_TELEMETRY_API_URL", + "DEFAULT_TELEMETRY_EXPERIMENT_ID", + "DEFAULT_TELEMETRY_LEVEL", + "MINIMAL_TELEMETRY_FIELDS", + "TELEMETRY_LEVEL_CONFIG_KEY", + "Telemetry", + "TelemetryContext", + "TelemetryCreate", + "TelemetryLevel", + "TelemetryLevelSource", + "TelemetrySettings", + "build_payload", + "parse_telemetry_level", + "post_private", + "post_public_summary", +] diff --git a/codecarbon/core/telemetry/client.py b/codecarbon/core/telemetry/client.py new file mode 100644 index 000000000..571ece940 --- /dev/null +++ b/codecarbon/core/telemetry/client.py @@ -0,0 +1,66 @@ +"""HTTP and API clients for product telemetry.""" + +from __future__ import annotations + +import dataclasses + +import requests + +from codecarbon.core.api_client import ApiClient +from codecarbon.core.telemetry.schemas import TelemetryCreate +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + + +def post_private(settings: TelemetrySettings, payload: dict) -> bool: + """POST private telemetry to ``/telemetry``; return True on HTTP 201.""" + headers = {"Content-Type": "application/json"} + if settings.api_key: + headers["x-api-token"] = settings.api_key + body = TelemetryCreate(**payload).model_dump(mode="json", exclude_none=True) + telemetry_url = f"{settings.api_url.rstrip('/')}/telemetry" + try: + response = requests.post( + url=telemetry_url, + json=body, + headers=headers, + timeout=2, + ) + except Exception: + logger.error("Telemetry request failed.", exc_info=True) + return False + if response.status_code == 201: + return True + if response.status_code == 404: + logger.warning( + "Telemetry API not found at %s (HTTP 404); Tier 1 not recorded.", + telemetry_url, + ) + else: + logger.error( + "Telemetry API %s: %s", + response.status_code, + response.text, + ) + return False + + +def post_public_summary( + settings: TelemetrySettings, + conf: dict, + emissions: EmissionsData, +) -> bool: + """Post public run summary via ``ApiClient`` (extensive tier only).""" + try: + api = ApiClient( + endpoint_url=settings.api_url, + experiment_id=settings.experiment_id, + api_key=settings.api_key, + conf=conf, + create_run_automatically=True, + ) + return bool(api.add_emission(dataclasses.asdict(emissions))) + except Exception as error: + logger.error(f"Public run summary failed (non-critical): {error}") + return False diff --git a/codecarbon/core/telemetry/collect.py b/codecarbon/core/telemetry/collect.py new file mode 100644 index 000000000..b3a45e74b --- /dev/null +++ b/codecarbon/core/telemetry/collect.py @@ -0,0 +1,340 @@ +"""Collect private product telemetry from tracker state.""" + +from __future__ import annotations + +import importlib.util +import os +import platform +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + +from codecarbon.core.cloud import get_env_cloud_details +from codecarbon.core.gpu import is_nvidia_system +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.output_methods.base_output import OutputMethod +from codecarbon.output_methods.emissions_data import EmissionsData + +FRAMEWORK_PACKAGES = ( + ("torch", "has_torch"), + ("transformers", "has_transformers"), + ("diffusers", "has_diffusers"), +) + +OUTPUT_METHOD_LABELS = { + OutputMethod.CSV: "file", + OutputMethod.API: "api", + OutputMethod.LOGGER: "logger", + OutputMethod.PROMETHEUS: "prometheus", + OutputMethod.LOGFIRE: "logfire", +} + +CI_ENV_VAR_LABELS = ( + ("GITHUB_ACTIONS", "github_actions"), + ("GITLAB_CI", "gitlab_ci"), + ("CIRCLECI", "circleci"), + ("JENKINS_URL", "jenkins"), + ("CI", "ci"), +) + +PACKAGE_MANAGER_ENV = ( + ("UV", "uv"), + ("POETRY_ACTIVE", "poetry"), + ("PIP_RUN", "pip"), +) + + +@dataclass +class TelemetryContext: + """Snapshot of tracker state used to build a telemetry payload.""" + + conf: dict[str, Any] + emissions: EmissionsData + hardware: list[Any] + resource_tracker: Any + output_methods: list[str] + tasks: dict[str, Any] + measure_power_secs: float | None + integration: str + + @classmethod + def from_tracker(cls, tracker: Any, emissions: EmissionsData) -> TelemetryContext: + """Build a context snapshot from an active emissions tracker.""" + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + methods = [ + OUTPUT_METHOD_LABELS[method] + for method in getattr(tracker, "_output_methods", None) or [] + if method in OUTPUT_METHOD_LABELS + ] + if getattr(tracker, "_emissions_endpoint", None): + methods.append("http") + + argv = " ".join(sys.argv) + if isinstance(tracker, OfflineEmissionsTracker): + integration = "offline_tracker" + elif "codecarbon" in argv and "monitor" in argv: + integration = "cli_monitor" + else: + integration = "library" + + return cls( + conf=getattr(tracker, "_conf", {}), + emissions=emissions, + hardware=getattr(tracker, "_hardware", []) or [], + resource_tracker=getattr(tracker, "_resource_tracker", None), + output_methods=methods, + tasks=getattr(tracker, "_tasks", {}) or {}, + measure_power_secs=getattr(tracker, "_measure_power_secs", None), + integration=integration, + ) + + +def _strip_empty(data: dict[str, Any]) -> dict[str, Any]: + return { + key: value for key, value in data.items() if value not in (None, "", [], {}) + } + + +def _env_label(env_vars: tuple[tuple[str, str], ...]) -> str | None: + return next((label for var, label in env_vars if os.environ.get(var)), None) + + +def _package_installed(name: str) -> bool: + return importlib.util.find_spec(name) is not None + + +def _round_coordinate(value: Any) -> float | None: + if value is None: + return None + return round(float(value), 1) + + +def _cloud_region( + emissions: EmissionsData, +) -> tuple[str | None, str | None, str | None]: + details = get_env_cloud_details() + raw_provider = raw_region = None + if details and details.get("metadata"): + provider = (details.get("provider") or "").lower() or None + metadata = details.get("metadata") or {} + if provider == "aws": + raw_region = metadata.get("region") + elif provider == "azure": + raw_region = (metadata.get("compute") or {}).get("location") + elif provider == "gcp": + zone = metadata.get("zone") or "" + parts = zone.split("/") + raw_region = parts[-1].rsplit("-", 1)[0] if parts else None + raw_provider = provider + + cloud_provider = emissions.cloud_provider or raw_provider + cloud_region = emissions.cloud_region or raw_region + region = emissions.region + if emissions.on_cloud == "Y" and cloud_region: + region = region or cloud_region + return cloud_provider, cloud_region, region + + +def _detect_python_env_type() -> str | None: + if os.environ.get("CONDA_DEFAULT_ENV"): + return "conda" + if os.environ.get("VIRTUAL_ENV"): + return "venv" + if sys.prefix != getattr(sys, "base_prefix", sys.prefix): + return "venv" + return "system" + + +def _detect_codecarbon_install_method() -> str | None: + try: + from importlib.metadata import distribution + + dist = distribution("codecarbon") + if getattr(dist, "editable", False): + return "editable" + installer = (dist.metadata.get("Installer") or "").lower() + if "uv" in installer: + return "uv" + if "pip" in installer: + return "pip" + except Exception: + pass + return None + + +def _detect_notebook_environment() -> str | None: + if os.environ.get("COLAB_GPU") is not None or "google.colab" in sys.modules: + return "colab" + try: + from IPython import get_ipython + + if "ZMQInteractiveShell" in get_ipython().__class__.__name__: + return "jupyter" + except Exception: + pass + return None + + +def _container_info() -> tuple[bool | None, str | None]: + if os.environ.get("KUBERNETES_SERVICE_HOST"): + return True, "kubernetes" + if os.path.exists("/.dockerenv"): + return True, "docker" + return None, None + + +def _detect_ide() -> str | None: + if os.environ.get("CURSOR_TRACE_ID") or os.environ.get("CURSOR_SESSION"): + return "cursor" + if os.environ.get("VSCODE_PID") or os.environ.get("TERM_PROGRAM") == "vscode": + return "vscode" + if os.environ.get("PYCHARM_HOSTED"): + return "pycharm" + return None + + +def _cudnn_version() -> str | None: + if not _package_installed("torch"): + return None + try: + import torch + + version = torch.backends.cudnn.version() + return str(version) if version is not None else None + except Exception: + return None + + +def _gpu_static_fields() -> dict[str, Any]: + if not is_nvidia_system(): + return {} + try: + import pynvml + + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + mem = pynvml.nvmlDeviceGetMemoryInfo(handle) + cuda_version = pynvml.nvmlSystemGetCudaDriverVersion_v2() + if isinstance(cuda_version, int): + cuda_version = f"{cuda_version // 1000}.{(cuda_version % 1000) // 10}" + return { + "gpu_memory_total_gb": mem.total / (1024**3), + "gpu_driver_version": pynvml.nvmlSystemGetDriverVersion(), + "cuda_version": cuda_version, + } + except Exception: + return {} + + +def _hardware_diagnostics(ctx: TelemetryContext) -> dict[str, Any]: + from codecarbon.core import cpu + + hardware_tracked: list[str] = [] + for item in ctx.hardware: + try: + hardware_tracked.append(item.description()) + except Exception: + pass + gpu_detection_method = None + if ctx.resource_tracker is not None: + gpu_tracker = getattr(ctx.resource_tracker, "gpu_tracker", None) + if gpu_tracker and gpu_tracker != "Unspecified": + gpu_detection_method = gpu_tracker + + rapl_available = cpu.is_rapl_available() if platform.system() == "Linux" else None + return { + "hardware_tracked": hardware_tracked or None, + "hardware_detection_success": bool(hardware_tracked), + "rapl_available": rapl_available, + "gpu_detection_method": gpu_detection_method, + "api_mode": "online" if "api" in ctx.output_methods else "offline", + } + + +def _minimal_payload(ctx: TelemetryContext, level: TelemetryLevel) -> dict[str, Any]: + """Fields allowed for ``telemetry_level=minimal`` (matches DB + MINIMAL_TELEMETRY_FIELDS).""" + conf = ctx.conf + emissions = ctx.emissions + cloud_provider, cloud_region, region = _cloud_region(emissions) + region = region or conf.get("region") + + payload = { + "timestamp": datetime.now(timezone.utc), + "telemetry_level": level.value, + "os": conf.get("os") or platform.platform(), + "country_name": emissions.country_name, + "country_iso_code": emissions.country_iso_code, + "region": region, + "cloud_provider": cloud_provider, + "cloud_region": cloud_region, + "longitude": _round_coordinate(conf.get("longitude", emissions.longitude)), + "latitude": _round_coordinate(conf.get("latitude", emissions.latitude)), + "cpu_count": conf.get("cpu_count"), + "cpu_physical_count": conf.get("cpu_physical_count"), + "cpu_model": conf.get("cpu_model"), + "cpu_architecture": platform.machine(), + "gpu_count": conf.get("gpu_count"), + "gpu_model": conf.get("gpu_model"), + "ram_total_size_gb": conf.get("ram_total_size"), + "python_version": conf.get("python_version") or platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_env_type": _detect_python_env_type(), + "codecarbon_version": conf.get("codecarbon_version"), + "codecarbon_install_method": _detect_codecarbon_install_method(), + "cudnn_version": _cudnn_version(), + **_gpu_static_fields(), + } + return _strip_empty(payload) + + +def _extensive_payload(ctx: TelemetryContext) -> dict[str, Any]: + """Extra fields stored only for ``telemetry_level=extensive``.""" + emissions = ctx.emissions + in_container, container_runtime = _container_info() + framework_fields = { + has_field: _package_installed(package) + for package, has_field in FRAMEWORK_PACKAGES + } + + return _strip_empty( + { + "tracking_mode": ctx.conf.get("tracking_mode"), + "decorator_vs_context": ctx.integration, + "output_methods": ctx.output_methods or None, + "task_tracking_used": bool(ctx.tasks), + "measure_power_interval_secs": ctx.measure_power_secs, + "python_package_manager": _env_label(PACKAGE_MANAGER_ENV), + "in_container": in_container, + "container_runtime": container_runtime, + "ci_environment": _env_label(CI_ENV_VAR_LABELS), + "notebook_environment": _detect_notebook_environment(), + "ide_used": _detect_ide(), + "duration_seconds": ( + float(emissions.duration) if emissions.duration else None + ), + "total_emissions_kg": emissions.emissions, + "emissions_rate_kg_per_sec": emissions.emissions_rate, + "energy_consumed_kwh": emissions.energy_consumed, + "cpu_energy_kwh": emissions.cpu_energy, + "gpu_energy_kwh": emissions.gpu_energy, + "ram_energy_kwh": emissions.ram_energy, + "cpu_utilization_avg": emissions.cpu_utilization_percent, + "gpu_utilization_avg": emissions.gpu_utilization_percent, + "ram_utilization_avg": emissions.ram_utilization_percent, + **framework_fields, + **_hardware_diagnostics(ctx), + } + ) + + +def build_payload( + ctx: TelemetryContext, + level: TelemetryLevel = TelemetryLevel.minimal, +) -> dict[str, Any]: + """Build a validated telemetry payload dict for ``POST /telemetry``.""" + payload = _minimal_payload(ctx, level) + if level == TelemetryLevel.extensive: + payload.update(_extensive_payload(ctx)) + return payload diff --git a/codecarbon/core/telemetry/dispatcher.py b/codecarbon/core/telemetry/dispatcher.py new file mode 100644 index 000000000..8692d8bd1 --- /dev/null +++ b/codecarbon/core/telemetry/dispatcher.py @@ -0,0 +1,65 @@ +"""Per-tracker telemetry dispatcher.""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from codecarbon.core.telemetry.client import post_private, post_public_summary +from codecarbon.core.telemetry.collect import TelemetryContext, build_payload +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.core.telemetry.settings import TelemetrySettings +from codecarbon.external.logger import logger +from codecarbon.output_methods.emissions_data import EmissionsData + +TELEMETRY_NOT_CONFIGURED_MESSAGE = ( + "telemetry_level not set explicitly; default %r. Minimal telemetry sends on each " + "stop. Set telemetry_level in .codecarbon.config, CODECARBON_TELEMETRY_LEVEL, " + "EmissionsTracker(telemetry_level=...), or: codecarbon telemetry set ." +) + + +class Telemetry: + """Per-tracker telemetry dispatcher.""" + + _default_warning_shown: ClassVar[bool] = False + + def __init__(self, settings: TelemetrySettings) -> None: + self.settings = settings + + @classmethod + def from_tracker(cls, tracker: Any) -> Telemetry: + """Build a dispatcher from tracker config state.""" + return cls( + TelemetrySettings.resolve( + config_file_conf=tracker._config_file_conf, + external_conf=tracker._external_conf, + override=getattr(tracker, "_telemetry_override", None), + ) + ) + + def warn_if_implicit(self) -> None: + """Log a one-time warning when telemetry tier was not set explicitly.""" + if self.settings.is_explicit or Telemetry._default_warning_shown: + return + logger.warning( + TELEMETRY_NOT_CONFIGURED_MESSAGE, + self.settings.level.value, + ) + Telemetry._default_warning_shown = True + + def send_at_stop(self, tracker: Any, emissions: EmissionsData) -> None: + """Send product telemetry at tracker ``stop()`` for the resolved tier.""" + if self.settings.level == TelemetryLevel.disabled: + return + if emissions.duration is not None and emissions.duration < 1: + logger.debug("Telemetry not sent: run shorter than 1 second.") + return + ctx = TelemetryContext.from_tracker(tracker, emissions) + payload = build_payload(ctx, level=self.settings.level) + post_private(self.settings, payload) + if self.settings.level == TelemetryLevel.extensive: + post_public_summary( + self.settings, + getattr(tracker, "_conf", {}), + emissions, + ) diff --git a/codecarbon/core/telemetry_schemas.py b/codecarbon/core/telemetry/schemas.py similarity index 96% rename from codecarbon/core/telemetry_schemas.py rename to codecarbon/core/telemetry/schemas.py index ea6249b65..81198ddc1 100644 --- a/codecarbon/core/telemetry_schemas.py +++ b/codecarbon/core/telemetry/schemas.py @@ -1,3 +1,8 @@ +"""Telemetry payload schemas aligned with carbonserver ``telemetry_sql_models.Telemetry``. + +ponytail: mirrored in ``carbonserver/api/schemas_telemetry.py`` for drift tests only. +""" + from datetime import datetime from enum import Enum from typing import List, Optional diff --git a/codecarbon/core/telemetry/settings.py b/codecarbon/core/telemetry/settings.py new file mode 100644 index 000000000..33ea6d125 --- /dev/null +++ b/codecarbon/core/telemetry/settings.py @@ -0,0 +1,121 @@ +"""Resolve telemetry tier and API settings from config and environment.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any, Literal + +from codecarbon.core.telemetry.schemas import TelemetryLevel +from codecarbon.external.logger import logger + +DEFAULT_TELEMETRY_API_URL = "https://api.codecarbon.io" +DEFAULT_TELEMETRY_API_KEY = "cpt_sDiIpdwl5BRUM2T6vIJrt2JjL-pB3b46v8cvpLwuroU" +DEFAULT_TELEMETRY_EXPERIMENT_ID = "d2d69403-1373-42b4-a2c1-09589aed4801" +DEFAULT_TELEMETRY_LEVEL = TelemetryLevel.minimal + +TELEMETRY_LEVEL_CONFIG_KEY = "telemetry_level" + +TelemetryLevelSource = Literal["override", "external", "file", "default"] + + +def parse_telemetry_level(raw: str | TelemetryLevel) -> TelemetryLevel: + """Parse a telemetry tier name or enum member.""" + if isinstance(raw, TelemetryLevel): + return raw + try: + return TelemetryLevel(str(raw).lower()) + except ValueError as error: + raise ValueError( + f"Invalid telemetry_level {raw!r}. Choose: disabled, minimal, or extensive." + ) from error + + +@dataclass(frozen=True) +class TelemetrySettings: + """Resolved telemetry tier and API connection settings.""" + + level: TelemetryLevel + source: TelemetryLevelSource + api_url: str + api_key: str + experiment_id: str + + @property + def is_explicit(self) -> bool: + """Return whether the user explicitly chose a telemetry tier.""" + return self.source != "default" + + @classmethod + def _with_connection( + cls, + merged: dict[str, Any], + level: TelemetryLevel, + source: TelemetryLevelSource, + ) -> TelemetrySettings: + return cls( + level=level, + source=source, + api_url=cls._resolve_api_url(merged), + api_key=cls._resolve_api_key(merged), + experiment_id=cls._resolve_experiment_id(merged), + ) + + @classmethod + def resolve( + cls, + *, + config_file_conf: dict[str, Any] | None = None, + external_conf: dict[str, Any] | None = None, + override: str | TelemetryLevel | None = None, + ) -> TelemetrySettings: + """Resolve tier (override > external > file > default minimal) and API settings.""" + merged = external_conf or {} + if override is not None: + raw = override + source: TelemetryLevelSource = "override" + elif merged.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None: + raw = merged[TELEMETRY_LEVEL_CONFIG_KEY] + source = "external" + elif ( + config_file_conf is not None + and config_file_conf.get(TELEMETRY_LEVEL_CONFIG_KEY) is not None + ): + raw = config_file_conf[TELEMETRY_LEVEL_CONFIG_KEY] + source = "file" + else: + return cls._with_connection(merged, DEFAULT_TELEMETRY_LEVEL, "default") + try: + level = parse_telemetry_level(raw) + except ValueError: + logger.error( + "Invalid telemetry_level provided; falling back to %r", + DEFAULT_TELEMETRY_LEVEL.value, + ) + level = DEFAULT_TELEMETRY_LEVEL + return cls._with_connection(merged, level, source) + + @staticmethod + def _resolve_api_url(external_conf: dict[str, Any]) -> str: + url = ( + external_conf.get("telemetry_api_url") + or external_conf.get("api_endpoint") + or os.environ.get("CODECARBON_TELEMETRY_API_URL") + ) + return (url or DEFAULT_TELEMETRY_API_URL).rstrip("/") + + @staticmethod + def _resolve_api_key(external_conf: dict[str, Any]) -> str: + key = ( + external_conf.get("telemetry_api_key") + or external_conf.get("api_key") + or os.environ.get("CODECARBON_TELEMETRY_API_KEY") + ) + return key or DEFAULT_TELEMETRY_API_KEY + + @staticmethod + def _resolve_experiment_id(external_conf: dict[str, Any]) -> str: + experiment_id = external_conf.get("telemetry_experiment_id") or os.environ.get( + "CODECARBON_TELEMETRY_EXPERIMENT_ID" + ) + return experiment_id or DEFAULT_TELEMETRY_EXPERIMENT_ID diff --git a/codecarbon/core/telemetry_client.py b/codecarbon/core/telemetry_client.py deleted file mode 100644 index 8dfdad05a..000000000 --- a/codecarbon/core/telemetry_client.py +++ /dev/null @@ -1,61 +0,0 @@ -import json -from typing import Optional, Union - -import requests - -from codecarbon.core.telemetry_schemas import TelemetryCreate -from codecarbon.external.logger import logger - - -class TelemetryClient: - """ - Client dedicated to sending CodeCarbon telemetry payloads. - """ - - def __init__( - self, - endpoint_url="https://api.codecarbon.io", - telemetry: Optional[Union[TelemetryCreate, dict]] = None, - ): - self.endpoint_url = endpoint_url.rstrip("/") - self.telemetry_url = self.endpoint_url + "/telemetry" - self.headers = {"Content-Type": "application/json"} - self.telemetry = self._validate_telemetry(telemetry) if telemetry else None - - def add_telemetry(self, telemetry: Optional[Union[TelemetryCreate, dict]] = None): - telemetry_payload = ( - self._validate_telemetry(telemetry) if telemetry else self.telemetry - ) - if telemetry_payload is None: - logger.error("TelemetryClient.add_telemetry() needs a telemetry payload") - return None - payload = telemetry_payload.model_dump(mode="json", exclude_none=True) - - try: - response = requests.post( - url=self.telemetry_url, - json=payload, - timeout=2, - headers=self.headers, - ) - if response.status_code != 201: - self._log_error(payload, response) - return None - return response.json() - except Exception as e: - logger.error(e, exc_info=True) - return None - - @staticmethod - def _validate_telemetry(telemetry: Union[TelemetryCreate, dict]) -> TelemetryCreate: - if isinstance(telemetry, TelemetryCreate): - return telemetry - return TelemetryCreate(**telemetry) - - def _log_error(self, payload, response): - logger.error( - f"TelemetryClient Error when calling the API on {self.telemetry_url} with : {json.dumps(payload)}" - ) - logger.error( - f"TelemetryClient API return http code {response.status_code} and answer : {response.text}" - ) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 88da92628..5e8015023 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -18,9 +18,14 @@ import psutil from codecarbon._version import __version__ -from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids +from codecarbon.core.config import ( + get_config_file_settings, + get_hierarchical_config, + normalize_gpu_ids, +) from codecarbon.core.emissions import Emissions from codecarbon.core.resource_tracker import ResourceTracker +from codecarbon.core.telemetry import Telemetry from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.geography import CloudMetadata, GeoMetadata @@ -403,6 +408,7 @@ def __init__( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -496,10 +502,16 @@ def __init__( (CPU + chipset + PCIe). When False, uses package domains which are more reliable. Note: psys can report higher values than CPU TDP and may be unreliable on older systems. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). + Overrides config file and ``CODECARBON_TELEMETRY_LEVEL`` when set. """ - # logger.info("base tracker init") self._external_conf = get_hierarchical_config() + self._config_file_conf = get_config_file_settings() + self._telemetry_override = ( + None if telemetry_level is _sentinel else telemetry_level + ) + self._telemetry = Telemetry.from_tracker(self) self._set_from_conf( force_carbon_intensity_g_co2e_kwh, "force_carbon_intensity_g_co2e_kwh", @@ -568,7 +580,6 @@ def __init__( self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1" ) - if self.force_carbon_intensity_g_co2e_kwh is not None: logger.info( f"Using forced carbon intensity: {self.force_carbon_intensity_g_co2e_kwh} gCO2e/kWh." @@ -583,8 +594,13 @@ def __init__( self._log_tracker_metadata() self._initialize_scheduler_state() self._initialize_emissions_context() + self._telemetry.warn_if_implicit() self._init_output_methods(api_key=self._api_key) + @suppress(Exception) + def _send_telemetry_at_stop(self, emissions_data: EmissionsData) -> None: + self._telemetry.send_at_stop(self, emissions_data) + def _init_output_methods(self, *, api_key: str = None): """ Prepare the different output methods based on ``self._output_methods``. @@ -891,6 +907,7 @@ def stop(self) -> Optional[float]: emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) + self._send_telemetry_at_stop(emissions_data) self._persist_data( total_emissions=emissions_data, @@ -1455,6 +1472,7 @@ def track_emissions( allow_multiple_runs: Optional[bool] = _sentinel, rapl_include_dram: Optional[bool] = _sentinel, rapl_prefer_psys: Optional[bool] = _sentinel, + telemetry_level: Optional[str] = _sentinel, ): """ Decorator that supports both `EmissionsTracker` and `OfflineEmissionsTracker` @@ -1536,6 +1554,7 @@ def track_emissions( When True, measures CPU package + DRAM. :param rapl_prefer_psys: Prefer psys over package domains for RAPL on Linux (default: False). When True, uses total platform power. + :param telemetry_level: Telemetry tier (``disabled``, ``minimal``, ``extensive``). :return: The decorated function """ @@ -1592,6 +1611,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) else: tracker = EmissionsTracker( @@ -1628,6 +1648,7 @@ def wrapped_fn(*args, **kwargs): allow_multiple_runs=allow_multiple_runs, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, + telemetry_level=telemetry_level, ) tracker.start() try: diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 39d600ec2..c649bd540 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -145,3 +145,9 @@ os.environ["HTTPS_PROXY"] = "http://0.0.0.0:0000" For more information, please read the [requests library proxy documentation](https://requests.readthedocs.io/en/latest/user/advanced/#proxies) + +## Product telemetry + +Optional library telemetry (`telemetry_level`: `disabled`, `minimal`, or `extensive`) is configured separately from dashboard API settings. Set it in `.codecarbon.config`, via `CODECARBON_TELEMETRY_LEVEL`, or with `EmissionsTracker(telemetry_level=...)` (argument wins). `minimal` sends private product telemetry at each `stop()`—see [Product telemetry](telemetry.md). + +See [Product telemetry](telemetry.md) for tiers, what is collected, and how to opt out. diff --git a/docs/how-to/telemetry.md b/docs/how-to/telemetry.md new file mode 100644 index 000000000..e0246a735 --- /dev/null +++ b/docs/how-to/telemetry.md @@ -0,0 +1,107 @@ +# Product telemetry + +CodeCarbon can send **optional private product telemetry** to help improve the library: hardware, environment, how the package is used, and per-run carbon/energy summaries. This is separate from sending **your** emissions to the [dashboard](cloud-api.md) with `save_to_api=True`. + +## Telemetry vs your dashboard data + +| | Product telemetry | Your emissions (`save_to_api`) | +|--|-------------------|--------------------------------| +| Purpose | Improve CodeCarbon (aggregate usage) | Your projects and experiments | +| Config | `telemetry_level`, `codecarbon telemetry` | `codecarbon config`, `experiment_id` | +| Default API target | Built-in telemetry project (private) | Your account / experiment | + +You can use one without the other. + +## Tiers + +| `telemetry_level` | Name | When | Transport | +|-------------------|------|------|-----------| +| `disabled` | — | — | Nothing | +| `minimal` | Private product telemetry | Each `stop()` | `POST /telemetry` (private) | +| `extensive` | Private telemetry + shared run summary | Each `stop()` | Same private `POST /telemetry` **and** `ApiClient` → `/emissions` | + +Tier is resolved in this order: + +1. **Tracker or CLI argument** — `EmissionsTracker(telemetry_level=...)` or `codecarbon monitor --telemetry-level ...` +2. **Config + environment** — `telemetry_level` in `.codecarbon.config`, then `CODECARBON_TELEMETRY_LEVEL` when both are set +3. **Default:** `minimal` + +## Lifecycle + +```text +EmissionsTracker.__init__ → collect hardware/geo (no POST) +EmissionsTracker.stop() → minimal: private POST only | extensive: private POST + /emissions +``` + +If the run lasts less than one second, telemetry is not sent. + +## Private telemetry — per run + +Both tiers POST to `/telemetry` at each `stop()`. The server schema defines what each tier may include: + +| Tier | Private `POST /telemetry` payload | +|------|-----------------------------------| +| `minimal` | Environment and hardware only (OS, Python, CPU/GPU/RAM, geo/cloud, CodeCarbon version) | +| `extensive` | Minimal fields **plus** run metrics, output methods, framework flags, usage diagnostics | + +**Minimal** includes rounded coordinates (1 decimal), cloud region, and hardware metadata. It does **not** include run emissions, energy, duration, or framework flags. + +**Extensive** adds run outcome (duration, emissions, energy, utilization), output methods, ML framework presence flags (booleans only, no package versions), CI/notebook/container/IDE hints, and integration context (`decorator_vs_context`: `library`, `cli_monitor`, or `offline_tracker`). + +Private telemetry does **not** include project names, experiment ids, API keys, file paths, executable/host hashes, or survey demographics. + +## `extensive` — additional public run summary + +**Also** posts a **run emissions summary** to the shared CodeCarbon telemetry experiment via `ApiClient` (`/runs` then `/emissions`). Endpoint, API key, and experiment id come from `telemetry_api_url` / `telemetry_api_key` / `telemetry_experiment_id` (or `CODECARBON_TELEMETRY_*` env vars), falling back to the built-in defaults and your `api_endpoint` / `api_key` when set. + +## Never collected + +- Project name, experiment id, run id, API keys +- Source code, file paths, hostnames +- Exact GPS coordinates, executable/host fingerprints (not in the telemetry schema) +- Voluntary [user survey](https://docs.google.com/forms/d/e/1FAIpQLSeQ5Tu_rdrpDhBJvh5R1-_iB4Ld-kgh6iNMjgaMXa8AEVPxqA/viewform) demographics (role, industry, experience) + +## Configure telemetry + +### Config file + +```ini +[codecarbon] +telemetry_level = minimal +``` + +### CLI + +```bash +codecarbon telemetry set minimal +codecarbon telemetry status +codecarbon monitor --telemetry-level disabled -- python train.py +``` + +### Python + +```python +from codecarbon import EmissionsTracker + +tracker = EmissionsTracker(telemetry_level="minimal") +tracker.start() +# ... +tracker.stop() +``` + +## Opt out + +```ini +[codecarbon] +telemetry_level = disabled +``` + +## First run without explicit configuration + +If you never set `telemetry_level`, CodeCarbon uses `minimal` and logs a **one-time warning** per Python session. Set `telemetry_level` explicitly to silence it. + +## Related + +- [Configure CodeCarbon](configuration.md) +- [CLI reference](../reference/cli.md#codecarbon-telemetry) +- [Cloud API & dashboard](cloud-api.md) diff --git a/docs/reference/api.md b/docs/reference/api.md index 15117b123..fd3a97e5e 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -12,6 +12,20 @@ Parameters can be set via `EmissionsTracker()`, `OfflineEmissionsTracker()`, the If you use `CUDA_VISIBLE_DEVICES` or `ROCR_VISIBLE_DEVICES` to set GPUs, CodeCarbon will automatically populate `gpu_ids`. Manual `gpu_ids` overrides this. +## Product telemetry + +Optional library telemetry is controlled by **`telemetry_level`** on the tracker (same parameter on `OfflineEmissionsTracker` and `@track_emissions`): + +| Value | Behavior | +|-------|----------| +| `disabled` | No product telemetry | +| `minimal` (default) | Private telemetry at each `stop()` | +| `extensive` | Private telemetry plus public run summary at each `stop()` | + +**Resolution order:** tracker argument → `.codecarbon.config` → `CODECARBON_TELEMETRY_LEVEL` → default `minimal`. The tracker argument overrides config and environment. + +This is separate from `save_to_api` (your dashboard experiment). See [Product telemetry](../how-to/telemetry.md). + ## EmissionsTracker / BaseEmissionsTracker `EmissionsTracker` and `OfflineEmissionsTracker` inherit from `BaseEmissionsTracker`. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1e93a588f..56ee7b909 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -45,6 +45,7 @@ Displays real-time emissions data for all processes on your machine. Press `Ctrl | `--offline` | flag | false | Run without internet access | | `--country-iso-code` | string | - | ISO 3166-1 alpha-3 country code (required in offline mode) | | `--log-level` | choice | INFO | Log level: DEBUG, INFO, WARNING, ERROR | +| `--telemetry-level` | string | - | One-run tier: `disabled`, `minimal`, or `extensive` | **Examples:** ```bash @@ -88,6 +89,34 @@ codecarbon monitor -- node app.js --port 8080 Same options as `codecarbon monitor` apply (see above). +### `codecarbon telemetry` + +Configure **product telemetry** (library usage metadata), separate from `codecarbon config` (dashboard org/project/experiment). + +**Usage:** + +```bash +codecarbon telemetry # interactive wizard +codecarbon telemetry set # disabled | minimal | extensive +codecarbon telemetry status # resolved tier and whether it was set explicitly +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--config PATH` | Use a specific `.codecarbon.config` (default: local then global) | + +See [Product telemetry](../how-to/telemetry.md) for what `minimal` collects and how to opt out. + +### `codecarbon monitor --telemetry-level` + +Override the telemetry tier for a single monitor run (does not update config). + +```bash +codecarbon monitor --telemetry-level disabled -- python train.py +``` + ### `codecarbon detect` Detect and print hardware information. diff --git a/mkdocs.yml b/mkdocs.yml index 19c451331..d167ec972 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -149,6 +149,7 @@ nav: - How-to Guides: - Examples: how-to/examples.md - Configure CodeCarbon: how-to/configuration.md + - Product telemetry: how-to/telemetry.md - Compare Model Efficiency: tutorials/comparing-model-efficiency.md - Dashboard & Visualization: - Use the Cloud API & Dashboard: how-to/cloud-api.md diff --git a/tests/cli/test_telemetry_cli.py b/tests/cli/test_telemetry_cli.py new file mode 100644 index 000000000..ef14b831a --- /dev/null +++ b/tests/cli/test_telemetry_cli.py @@ -0,0 +1,261 @@ +"""Tests for codecarbon telemetry CLI commands.""" + +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +import typer +from typer.testing import CliRunner + +from codecarbon.cli import main as cli_main +from codecarbon.cli.telemetry_cli import ( + normalize_telemetry_level, + pick_config_path_interactive, + resolve_config_path, + telemetry_app, + write_telemetry_level, +) + + +def test_normalize_telemetry_level_accepts_valid_values(): + assert normalize_telemetry_level("MINIMAL") == "minimal" + assert normalize_telemetry_level("disabled") == "disabled" + + +def test_normalize_telemetry_level_rejects_invalid(): + with pytest.raises(typer.BadParameter): + normalize_telemetry_level("bogus") + + +def test_telemetry_set_writes_config(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + result = runner.invoke( + telemetry_app, + ["set", "disabled", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "telemetry_level = disabled" in result.output + content = config_path.read_text() + assert "telemetry_level = disabled" in content + + +def test_telemetry_status_reports_stored_level(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / ".codecarbon.config" + config_path.write_text("[codecarbon]\ntelemetry_level = extensive\n") + result = runner.invoke( + telemetry_app, + ["status", "--config", str(config_path)], + ) + assert result.exit_code == 0 + assert "extensive" in result.output + assert "Explicitly configured: True" in result.output + + +def test_telemetry_status_merged_matches_tracker_precedence(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + result = runner.invoke(telemetry_app, ["status"]) + assert result.exit_code == 0 + assert "Resolved tier: disabled" in result.output + assert "merged" in result.output + + +def test_monitor_passes_telemetry_level_override(monkeypatch): + from codecarbon.cli import monitor as monitor_module + + captured = {} + + class FakeTracker: + def __init__(self, **kwargs): + captured.update(kwargs) + self._conf = {"output_file": "emissions.csv"} + + def start(self): + return None + + def stop(self): + return 0.0 + + monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + [ + "monitor", + "--no-api", + "--telemetry-level", + "disabled", + "--", + "echo", + "ok", + ], + ) + assert result.exit_code == 0 + assert captured.get("telemetry_level") == "disabled" + + +def test_monitor_rejects_invalid_telemetry_level(): + runner = CliRunner() + result = runner.invoke( + cli_main.codecarbon, + ["monitor", "--telemetry-level", "invalid"], + ) + assert result.exit_code != 0 + + +def test_telemetry_status_missing_config_file(): + runner = CliRunner() + with tempfile.TemporaryDirectory() as tmp: + missing = Path(tmp) / "missing.config" + result = runner.invoke( + telemetry_app, + ["status", "--config", str(missing)], + ) + assert result.exit_code == 0 + assert "Config file not found" in result.output + assert "not explicit" in result.output + + +def test_telemetry_status_shows_implicit_warning(): + runner = CliRunner() + with patch( + "codecarbon.cli.telemetry_cli.get_config_file_settings", + return_value={}, + ): + with patch( + "codecarbon.cli.telemetry_cli.get_hierarchical_config", + return_value={}, + ): + result = runner.invoke(telemetry_app, ["status"]) + assert "Explicitly configured: False" in result.output + assert "Minimal telemetry will be sent" in result.output + + +def test_resolve_config_path_creates_explicit_file(): + with tempfile.TemporaryDirectory() as tmp: + config_path = Path(tmp) / "custom.config" + resolved = resolve_config_path(config_path, create=True) + assert resolved == config_path.resolve() + assert config_path.exists() + + +def test_resolve_config_path_prefers_local_config(tmp_path, monkeypatch): + local_path = tmp_path / ".codecarbon.config" + local_path.write_text("[codecarbon]\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert resolve_config_path(None) == local_path.resolve() + + +def test_resolve_config_path_uses_global_when_local_missing(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + global_path = home / ".codecarbon.config" + global_path.write_text("[codecarbon]\n", encoding="utf-8") + workdir = tmp_path / "work" + workdir.mkdir() + monkeypatch.chdir(workdir) + monkeypatch.setattr(Path, "home", lambda: home) + assert resolve_config_path(None) == global_path.resolve() + + +def test_write_telemetry_level_creates_missing_file(tmp_path): + config_path = tmp_path / "nested" / ".codecarbon.config" + write_telemetry_level(config_path, "minimal") + assert config_path.exists() + assert "telemetry_level = minimal" in config_path.read_text() + + +def test_pick_config_path_interactive_create_new(): + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = "Create new config file" + with patch( + "codecarbon.cli.telemetry_cli.create_new_config_file", + return_value=Path("/tmp/new.config"), + ) as mock_create: + path = pick_config_path_interactive() + mock_create.assert_called_once() + assert path == Path("/tmp/new.config") + + +def test_resolve_config_path_creates_local_in_cwd(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + monkeypatch.setattr(Path, "home", lambda: home) + resolved = resolve_config_path(None, create=True) + assert resolved == (work / ".codecarbon.config").resolve() + assert resolved.exists() + + +def test_pick_config_path_lists_existing_configs(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + global_path = home / ".codecarbon.config" + global_path.write_text("[codecarbon]\n", encoding="utf-8") + work = tmp_path / "work" + work.mkdir() + local_path = work / ".codecarbon.config" + local_path.write_text("[codecarbon]\n", encoding="utf-8") + monkeypatch.chdir(work) + monkeypatch.setattr(Path, "home", lambda: home) + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = str(local_path) + path = pick_config_path_interactive() + assert path == local_path.resolve() + + +def test_telemetry_default_command_runs_interactive_wizard(tmp_path): + config_path = tmp_path / ".codecarbon.config" + config_path.write_text("[codecarbon]\n", encoding="utf-8") + runner = CliRunner() + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = "disabled" + result = runner.invoke(telemetry_app, ["--config", str(config_path)]) + assert result.exit_code == 0 + assert "telemetry_level = disabled" in config_path.read_text() + + +def test_telemetry_interactive_exits_when_selection_cancelled(tmp_path): + config_path = tmp_path / ".codecarbon.config" + config_path.write_text("[codecarbon]\n", encoding="utf-8") + runner = CliRunner() + with patch("codecarbon.cli.telemetry_cli.questionary.select") as mock_select: + mock_select.return_value.ask.return_value = None + result = runner.invoke(telemetry_app, ["--config", str(config_path)]) + assert result.exit_code == 0 + assert "telemetry_level" not in config_path.read_text() + + +def test_telemetry_interactive_prompts_for_config_path(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + runner = CliRunner() + select = MagicMock() + select.ask.return_value = "minimal" + with patch( + "codecarbon.cli.telemetry_cli.pick_config_path_interactive", + return_value=tmp_path / ".codecarbon.config", + ): + with patch( + "codecarbon.cli.telemetry_cli.questionary.select", return_value=select + ): + result = runner.invoke(telemetry_app, []) + config_path = tmp_path / ".codecarbon.config" + assert result.exit_code == 0 + assert config_path.exists() + assert "telemetry_level = minimal" in config_path.read_text() diff --git a/tests/test_api_call.py b/tests/test_api_call.py index 39822ece7..043f9c5a0 100644 --- a/tests/test_api_call.py +++ b/tests/test_api_call.py @@ -4,7 +4,7 @@ import requests_mock -from codecarbon.core.api_client import ApiClient +from codecarbon.core.api_client import ApiClient, _round_coordinate from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate from codecarbon.output import EmissionsData @@ -26,6 +26,11 @@ class TestApi(unittest.TestCase): + def test_round_coordinate_handles_none_and_values(self): + self.assertEqual(_round_coordinate(None), 0.0) + self.assertEqual(_round_coordinate(-7.61743), -7.6) + self.assertEqual(_round_coordinate(33.58229, decimals=2), 33.58) + def test_get_headers_prefers_api_key_over_access_token(self): api = ApiClient( endpoint_url="http://test.com", @@ -253,6 +258,22 @@ def test_add_emission_returns_false_on_unsuccessful_post(self): ) ) + def test_create_run_handles_none_coordinates(self): + conf_with_none_coords = {**conf, "longitude": None, "latitude": None} + with requests_mock.Mocker() as m: + m.post("http://test.com/runs", json={"id": "run-id"}, status_code=201) + api = ApiClient( + endpoint_url="http://test.com", + experiment_id="experiment_id", + api_key="Toto", + conf=conf_with_none_coords, + create_run_automatically=False, + ) + run_id = api._create_run("experiment_id") + self.assertEqual(run_id, "run-id") + self.assertEqual(m.last_request.json()["longitude"], 0.0) + self.assertEqual(m.last_request.json()["latitude"], 0.0) + def test_create_run_returns_none_on_unsuccessful_status(self): with requests_mock.Mocker() as m: m.post("http://test.com/runs", text="bad", status_code=400) diff --git a/tests/test_config.py b/tests/test_config.py index ef66f66f8..e2ba9fffe 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +import configparser import os import unittest from textwrap import dedent @@ -20,6 +21,19 @@ from tests.testutils import get_custom_mock_open +def _file_settings_from_ini(global_conf: str, local_conf: str) -> dict[str, str]: + """Merge mocked global and local ``[codecarbon]`` sections like config files do.""" + merged: dict[str, str] = {} + for text in (global_conf, local_conf): + if not text.strip(): + continue + parser = configparser.ConfigParser() + parser.read_string(text) + if "codecarbon" in parser: + merged.update(dict(parser["codecarbon"])) + return merged + + class TestConfig(unittest.TestCase): def setUp(self): self._original_environ = os.environ.copy() @@ -27,6 +41,12 @@ def setUp(self): "CODECARBON_API_KEY", "CODECARBON_EXPERIMENT_ID", "CODECARBON_API_ENDPOINT", + "CODECARBON_TELEMETRY", + "CODECARBON_TELEMETRY_LEVEL", + "CODECARBON_TELEMETRY_PROJECT_TOKEN", + "CODECARBON_TELEMETRY_API_URL", + "CODECARBON_TELEMETRY_API_KEY", + "CODECARBON_TELEMETRY_EXPERIMENT_ID", "codecarbon_api_key", "codecarbon_experiment_id", "codecarbon_api_endpoint", @@ -119,7 +139,8 @@ def test_read_confs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -160,7 +181,8 @@ def test_read_confs_and_parse_envs(self): ) with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + "codecarbon.core.config.get_config_file_settings", + return_value=_file_settings_from_ini(global_conf, local_conf), ): conf = dict(get_hierarchical_config()) target = { @@ -175,12 +197,7 @@ def test_read_confs_and_parse_envs(self): self.assertDictEqual(conf, target) def test_empty_conf(self): - global_conf = "" - local_conf = "" - - with patch( - "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) - ): + with patch("codecarbon.core.config.get_config_file_settings", return_value={}): conf = dict(get_hierarchical_config()) # allow_multiple_runs is set in pytest.ini and not mocked, so it's visible here. target = {"allow_multiple_runs": "True"} diff --git a/tests/test_config_file_settings.py b/tests/test_config_file_settings.py new file mode 100644 index 000000000..6a155ef7e --- /dev/null +++ b/tests/test_config_file_settings.py @@ -0,0 +1,51 @@ +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from codecarbon.core.config import get_config_file_settings, get_hierarchical_config + + +class TestGetConfigFileSettings(unittest.TestCase): + def test_returns_empty_when_no_config_files(self): + with patch("codecarbon.core.config._config_file_paths") as mock_paths: + mock_paths.return_value = ("/nonexistent/global", "/nonexistent/local") + settings = get_config_file_settings() + self.assertEqual(settings, {}) + + def test_local_overrides_global_telemetry_level(self): + with tempfile.TemporaryDirectory() as tmp: + global_path = Path(tmp) / "global.config" + local_path = Path(tmp) / "local.config" + global_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + local_path.write_text("[codecarbon]\ntelemetry_level = disabled\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=(str(global_path), str(local_path)), + ): + settings = get_config_file_settings() + self.assertEqual(settings["telemetry_level"], "disabled") + + def test_hierarchical_config_includes_env_but_file_settings_do_not(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text("[codecarbon]\ntelemetry_level = minimal\n") + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + file_settings = get_config_file_settings() + hierarchical = get_hierarchical_config() + self.assertEqual(file_settings.get("telemetry_level"), "minimal") + self.assertNotIn("telemetry", file_settings) + self.assertEqual(hierarchical.get("telemetry"), "disabled") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index 37f1000de..95f285dcc 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -36,7 +36,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" if sys.platform == "darwin": @@ -74,7 +74,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 07adf403c..4b9b4b4ec 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -18,7 +18,7 @@ def heavy_computation(run_time_secs: float = 3): pass -empty_conf = "[codecarbon]" +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" class TestOfflineEmissionsTracker(unittest.TestCase): @@ -32,7 +32,8 @@ def setUp(self) -> None: # ./.codecarbon.config so that the user's local configuration does not # alter tests patcher = mock.patch( - "builtins.open", new_callable=get_custom_mock_open(empty_conf, empty_conf) + "builtins.open", + new_callable=get_custom_mock_open(disabled_conf, disabled_conf), ) self.addCleanup(patcher.stop) patcher.start() diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 000000000..9244579df --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,226 @@ +import sys +import tempfile +import unittest +from unittest.mock import ANY, MagicMock, patch + +from codecarbon.core.telemetry import Telemetry +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from codecarbon.output_methods.emissions_data import EmissionsData +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") + +disabled_conf = "[codecarbon]\ntelemetry_level = disabled\n" +minimal_conf = "[codecarbon]\ntelemetry_level = minimal\n" +extensive_conf = "[codecarbon]\ntelemetry_level = extensive\n" + + +class TestTelemetryTiersAtStop(unittest.TestCase): + def _emissions(self) -> EmissionsData: + return EmissionsData( + timestamp="2020-01-01T00:00:00", + project_name="test", + run_id="run-1", + experiment_id="exp-1", + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="cpu", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) + + def test_tier1_posts_private_telemetry(self): + tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None + tracker._conf = {"os": "Linux", "tracking_mode": "machine"} + tracker._hardware = [] + tracker._resource_tracker = None + tracker._save_to_api = False + tracker._save_to_file = False + tracker._save_to_logger = False + tracker._emissions_endpoint = None + tracker._save_to_prometheus = False + tracker._save_to_logfire = False + tracker._tasks = {} + tracker._measure_power_secs = 15 + emissions = self._emissions() + telemetry = Telemetry.from_tracker(tracker) + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + telemetry.send_at_stop(tracker, emissions) + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertNotIn("total_emissions_kg", payload) + + def test_tier1_skips_short_duration_at_dispatcher(self): + tracker = MagicMock() + tracker._config_file_conf = {} + tracker._external_conf = {} + tracker._telemetry_override = None + emissions = self._emissions() + emissions.duration = 0.5 + telemetry = Telemetry.from_tracker(tracker) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + telemetry.send_at_stop(tracker, emissions) + mock_post.assert_not_called() + + def test_tier2_uses_api_client(self): + tracker = MagicMock() + tracker._conf = {"os": "Linux"} + tracker._config_file_conf = {"telemetry_level": "extensive"} + tracker._external_conf = {} + tracker._telemetry_override = None + emissions = self._emissions() + telemetry = Telemetry.from_tracker(tracker) + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ): + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + telemetry.send_at_stop(tracker, emissions) + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + mock_api_cls.assert_called_with( + endpoint_url=ANY, + experiment_id=ANY, + api_key=ANY, + conf=tracker._conf, + create_run_automatically=True, + ) + + +@mock_platform_cli_setup +class TestTrackerTelemetry(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory() + self.patcher = None + Telemetry._default_warning_shown = False + + def tearDown(self) -> None: + if self.patcher: + self.patcher.stop() + self.temp_dir.cleanup() + Telemetry._default_warning_shown = False + + def _start_config_mock(self, conf: str) -> None: + self.patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self.patcher.start() + + def test_emissions_tracker_does_not_send_telemetry_on_init(self, mock_cli_setup): + self._start_config_mock(minimal_conf) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + mock_post.assert_not_called() + + def test_emissions_tracker_sends_telemetry_on_stop_when_minimal( + self, mock_cli_setup + ): + self._start_config_mock(minimal_conf) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + payload = mock_post.call_args[0][1] + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertNotIn("total_emissions_kg", payload) + + def test_emissions_tracker_skips_telemetry_when_disabled(self, mock_cli_setup): + self._start_config_mock(disabled_conf) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_not_called() + + def test_tier2_sends_tier1_and_api_client_on_stop(self, mock_cli_setup): + self._start_config_mock(extensive_conf) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + + def test_offline_tracker_sends_on_stop(self, mock_cli_setup): + self._start_config_mock(minimal_conf) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_client.py b/tests/test_telemetry_client.py index 14ade8b7f..2ca7581f7 100644 --- a/tests/test_telemetry_client.py +++ b/tests/test_telemetry_client.py @@ -1,29 +1,28 @@ import unittest +from unittest.mock import patch import requests_mock from pydantic import ValidationError -from codecarbon.core.telemetry_client import TelemetryClient -from codecarbon.core.telemetry_schemas import TelemetryCreate - - -class TestTelemetryClient(unittest.TestCase): - def test_init_sets_up_client_without_calling_api(self): - with requests_mock.Mocker() as m: - client = TelemetryClient( - endpoint_url="http://test.com/", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - }, - ) - - self.assertEqual(client.endpoint_url, "http://test.com") - self.assertEqual(client.telemetry_url, "http://test.com/telemetry") - self.assertIsInstance(client.telemetry, TelemetryCreate) - self.assertEqual(m.call_count, 0) +from codecarbon.core.telemetry import ( + TelemetrySettings, + post_private, + post_public_summary, +) +from codecarbon.output_methods.emissions_data import EmissionsData + + +class TestPostPrivate(unittest.TestCase): + def _settings(self, api_url: str = "http://test.com", api_key: str | None = None): + return TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url=api_url, + api_key=api_key or TelemetrySettings.resolve().api_key, + experiment_id=TelemetrySettings.resolve().experiment_id, + ) - def test_add_telemetry_posts_configured_payload(self): + def test_post_private_sends_validated_payload(self): telemetry = { "timestamp": "2026-05-03T12:00:00+00:00", "telemetry_level": "minimal", @@ -36,15 +35,9 @@ def test_add_telemetry_posts_configured_payload(self): json="f52fe339-164d-4c2b-a8c0-f562dfce066d", status_code=201, ) - client = TelemetryClient( - endpoint_url="http://test.com", telemetry=telemetry - ) + result = post_private(self._settings(), telemetry) - actual_telemetry_id = client.add_telemetry() - - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) + self.assertTrue(result) self.assertEqual(m.call_count, 1) self.assertEqual( m.last_request.json(), @@ -54,51 +47,148 @@ def test_add_telemetry_posts_configured_payload(self): }, ) - def test_add_telemetry_posts_call_payload(self): - telemetry = TelemetryCreate( - timestamp="2026-05-03T12:00:00+00:00", - telemetry_level="minimal", - os="Linux-5.10.0-x86_64", - ) + def test_post_private_rejects_invalid_payload(self): + with self.assertRaises(ValidationError): + post_private( + self._settings(), + { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + "unknown_field": "value", + }, + ) + def test_post_private_logs_warning_on_404(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } with requests_mock.Mocker() as m: m.post( "http://test.com/telemetry", - json="f52fe339-164d-4c2b-a8c0-f562dfce066d", + text='{"detail":"Not Found"}', + status_code=404, + ) + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_private(self._settings(), telemetry) + self.assertFalse(result) + mock_logger.warning.assert_called_once() + + def test_post_private_sends_api_key_header_when_configured(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + settings = TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id=TelemetrySettings.resolve().experiment_id, + ) + with requests_mock.Mocker() as m: + m.post( + "http://test.com/telemetry", + json="telemetry-id", status_code=201, ) - client = TelemetryClient(endpoint_url="http://test.com") + post_private(settings, telemetry) + self.assertEqual(m.last_request.headers["x-api-token"], "cpt_test_key") - actual_telemetry_id = client.add_telemetry(telemetry) + def test_post_private_returns_false_on_request_error(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.side_effect = ConnectionError("network down") + with patch("codecarbon.core.telemetry.client.logger"): + result = post_private(self._settings(), telemetry) + self.assertFalse(result) + + +class TestPostPublicSummary(unittest.TestCase): + def _settings(self) -> TelemetrySettings: + return TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id="00000000-0000-0000-0000-000000000001", + ) - self.assertEqual( - actual_telemetry_id, "f52fe339-164d-4c2b-a8c0-f562dfce066d" - ) - self.assertEqual(m.call_count, 1) - self.assertEqual( - m.last_request.json(), - { - "timestamp": "2026-05-03T12:00:00Z", - "telemetry_level": "minimal", - "os": "Linux-5.10.0-x86_64", - }, - ) + def _emissions(self) -> EmissionsData: + return EmissionsData( + timestamp="2026-05-03T12:00:00+00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.001, + emissions_rate=0.0001, + cpu_power=0.0, + gpu_power=0.0, + ram_power=0.0, + cpu_energy=0.0, + gpu_energy=0.0, + ram_energy=0.0, + energy_consumed=0.01, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="2.0", + cpu_count=1.0, + cpu_model="cpu", + gpu_count=0.0, + gpu_model="", + longitude=0.0, + latitude=0.0, + ram_total_size=8.0, + tracking_mode="machine", + ) - def test_init_rejects_invalid_telemetry_without_calling_api(self): + def test_post_public_summary_returns_true_on_success(self): + with patch("codecarbon.core.telemetry.client.ApiClient") as mock_client: + mock_client.return_value.add_emission.return_value = True + result = post_public_summary(self._settings(), {}, self._emissions()) + self.assertTrue(result) + + def test_post_public_summary_returns_false_on_error(self): + with patch( + "codecarbon.core.telemetry.client.ApiClient", + side_effect=RuntimeError("api down"), + ): + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_public_summary(self._settings(), {}, self._emissions()) + self.assertFalse(result) + mock_logger.error.assert_called_once() + + +class TestPostPrivateErrors(unittest.TestCase): + def test_post_private_logs_error_on_unexpected_status(self): + telemetry = { + "timestamp": "2026-05-03T12:00:00+00:00", + "telemetry_level": "minimal", + } + settings = TelemetrySettings( + level=TelemetrySettings.resolve().level, + source="default", + api_url="http://test.com", + api_key="cpt_test_key", + experiment_id=TelemetrySettings.resolve().experiment_id, + ) with requests_mock.Mocker() as m: - with self.assertRaises(ValidationError): - TelemetryClient( - endpoint_url="http://test.com", - telemetry={ - "timestamp": "2026-05-03T12:00:00+00:00", - "telemetry_level": "minimal", - "total_emissions_kg": 0.42, - }, - ) - - self.assertEqual(m.call_count, 0) - - def test_add_telemetry_returns_none_without_payload(self): - client = TelemetryClient(endpoint_url="http://test.com") - - self.assertIsNone(client.add_telemetry()) + m.post("http://test.com/telemetry", text="server error", status_code=500) + with patch("codecarbon.core.telemetry.client.logger") as mock_logger: + result = post_private(settings, telemetry) + self.assertFalse(result) + mock_logger.error.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_collect.py b/tests/test_telemetry_collect.py new file mode 100644 index 000000000..64deac26e --- /dev/null +++ b/tests/test_telemetry_collect.py @@ -0,0 +1,288 @@ +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +from codecarbon.core.telemetry import TelemetryContext, TelemetryLevel, build_payload +from codecarbon.core.telemetry.schemas import ( + MINIMAL_TELEMETRY_FIELDS, + TelemetryBase, + TelemetryCreate, +) +from codecarbon.output_methods.emissions_data import EmissionsData + + +def _sample_emissions(**overrides): + base = dict( + timestamp="2026-01-01T00:00:00", + project_name="p", + run_id="r", + experiment_id="e", + duration=10.0, + emissions=0.5, + emissions_rate=0.05, + cpu_power=1.0, + gpu_power=2.0, + ram_power=0.5, + cpu_energy=0.01, + gpu_energy=0.02, + ram_energy=0.001, + energy_consumed=0.031, + water_consumed=0.0, + country_name="France", + country_iso_code="FRA", + region="idf", + cloud_provider="", + cloud_region="", + os="Linux", + python_version="3.11", + codecarbon_version="3.0", + cpu_count=4, + cpu_model="cpu", + gpu_count=1, + gpu_model="gpu", + longitude=0.0, + latitude=0.0, + ram_total_size=16.0, + tracking_mode="machine", + ) + base.update(overrides) + return EmissionsData(**base) + + +def _tracker_context(**overrides) -> TelemetryContext: + conf = overrides.pop("conf", {"codecarbon_version": "3.0"}) + emissions = overrides.pop("emissions", _sample_emissions()) + output_methods = overrides.pop("output_methods", []) + return TelemetryContext( + conf=conf, + emissions=emissions, + hardware=overrides.pop("hardware", []), + resource_tracker=overrides.pop("resource_tracker", None), + output_methods=output_methods, + tasks=overrides.pop("tasks", {}), + measure_power_secs=overrides.pop("measure_power_secs", 15), + integration=overrides.pop("integration", "library"), + ) + + +class TestTelemetryCollect(unittest.TestCase): + def test_build_payload_minimal_omits_run_metrics(self): + ctx = _tracker_context( + conf={ + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + }, + output_methods=["file"], + ) + with patch( + "codecarbon.core.telemetry.collect._package_installed", + side_effect=lambda name: name == "torch", + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + + self.assertEqual(payload["telemetry_level"], "minimal") + self.assertNotIn("total_emissions_kg", payload) + self.assertNotIn("has_torch", payload) + self.assertNotIn("output_methods", payload) + + def test_build_payload_extensive_includes_run_and_framework_flags(self): + ctx = _tracker_context( + conf={ + "os": "Linux", + "codecarbon_version": "3.0", + "cpu_count": 4, + "tracking_mode": "machine", + }, + output_methods=["file"], + ) + with patch( + "codecarbon.core.telemetry.collect._package_installed", + side_effect=lambda name: name == "torch", + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + + self.assertEqual(payload["telemetry_level"], "extensive") + self.assertEqual(payload["total_emissions_kg"], 0.5) + self.assertEqual(payload["duration_seconds"], 10.0) + self.assertTrue(payload["has_torch"]) + self.assertIn("file", payload["output_methods"]) + + def test_build_payload_omits_framework_versions(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0", "hardware": ["cpu"]}) + with patch( + "codecarbon.core.telemetry.collect._package_installed", + return_value=True, + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + + self.assertEqual(payload["telemetry_level"], "extensive") + self.assertTrue(payload["has_torch"]) + self.assertNotIn("torch_version", payload) + + def test_build_payload_uses_resolved_level(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertEqual(payload["telemetry_level"], "extensive") + + def test_minimal_payload_passes_schema_validation(self): + ctx = _tracker_context(conf={"os": "Linux", "codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.minimal) + TelemetryCreate(**payload) + + def test_extensive_payload_passes_schema_validation(self): + ctx = _tracker_context(conf={"os": "Linux", "codecarbon_version": "3.0"}) + payload = build_payload(ctx, level=TelemetryLevel.extensive) + TelemetryCreate(**payload) + + def test_payload_keys_are_schema_fields(self): + ctx = _tracker_context(conf={"codecarbon_version": "3.0"}) + for level in (TelemetryLevel.minimal, TelemetryLevel.extensive): + payload = build_payload(ctx, level=level) + self.assertTrue(set(payload).issubset(TelemetryBase.model_fields)) + + def test_minimal_fields_are_schema_subset(self): + self.assertTrue(MINIMAL_TELEMETRY_FIELDS.issubset(TelemetryBase.model_fields)) + + def test_cloud_region_from_aws_metadata(self): + emissions = _sample_emissions(on_cloud="Y", cloud_region="", region="") + ctx = _tracker_context(emissions=emissions) + with patch( + "codecarbon.core.telemetry.collect.get_env_cloud_details", + return_value={"provider": "aws", "metadata": {"region": "eu-west-1"}}, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["cloud_provider"], "aws") + self.assertEqual(payload["cloud_region"], "eu-west-1") + + def test_cloud_region_from_azure_metadata(self): + emissions = _sample_emissions(on_cloud="Y", cloud_region="", region="") + ctx = _tracker_context(emissions=emissions) + with patch( + "codecarbon.core.telemetry.collect.get_env_cloud_details", + return_value={ + "provider": "azure", + "metadata": {"compute": {"location": "westeurope"}}, + }, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["cloud_provider"], "azure") + self.assertEqual(payload["cloud_region"], "westeurope") + + def test_cloud_region_from_gcp_metadata(self): + emissions = _sample_emissions(on_cloud="Y", cloud_region="", region="") + ctx = _tracker_context(emissions=emissions) + with patch( + "codecarbon.core.telemetry.collect.get_env_cloud_details", + return_value={ + "provider": "gcp", + "metadata": {"zone": "projects/p/zones/europe-west1-b"}, + }, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["cloud_provider"], "gcp") + self.assertEqual(payload["cloud_region"], "europe-west1") + + def test_extensive_payload_includes_environment_hints(self): + ctx = _tracker_context(output_methods=["api"]) + with patch.dict( + os.environ, + { + "CONDA_DEFAULT_ENV": "base", + "KUBERNETES_SERVICE_HOST": "10.0.0.1", + "CURSOR_TRACE_ID": "trace-1", + "COLAB_GPU": "0", + }, + clear=False, + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertEqual(payload["python_env_type"], "conda") + self.assertEqual(payload["api_mode"], "online") + self.assertTrue(payload["in_container"]) + self.assertEqual(payload["container_runtime"], "kubernetes") + self.assertEqual(payload["ide_used"], "cursor") + self.assertEqual(payload["notebook_environment"], "colab") + + def test_from_tracker_detects_cli_monitor(self): + tracker = MagicMock() + tracker._conf = {"codecarbon_version": "3.0"} + tracker._hardware = [] + tracker._resource_tracker = None + tracker._output_methods = [] + tracker._emissions_endpoint = None + tracker._tasks = {} + tracker._measure_power_secs = 15 + with patch.object(sys, "argv", ["codecarbon", "monitor", "train.py"]): + ctx = TelemetryContext.from_tracker(tracker, _sample_emissions()) + self.assertEqual(ctx.integration, "cli_monitor") + + def test_hardware_diagnostics_lists_tracked_hardware(self): + hardware = MagicMock() + hardware.description.return_value = "CPU: test" + ctx = _tracker_context( + hardware=[hardware], + resource_tracker=MagicMock(gpu_tracker="pynvml"), + ) + with patch( + "codecarbon.core.telemetry.collect.platform.system", return_value="Linux" + ): + with patch( + "codecarbon.core.cpu.is_rapl_available", + return_value=True, + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertIn("CPU: test", payload["hardware_tracked"]) + self.assertTrue(payload["hardware_detection_success"]) + self.assertTrue(payload["rapl_available"]) + self.assertEqual(payload["gpu_detection_method"], "pynvml") + + def test_gpu_static_fields_when_nvidia_available(self): + ctx = _tracker_context() + mock_mem = MagicMock(total=8 * 1024**3) + mock_pynvml = MagicMock() + mock_pynvml.nvmlDeviceGetMemoryInfo.return_value = mock_mem + mock_pynvml.nvmlSystemGetCudaDriverVersion_v2.return_value = 12040 + mock_pynvml.nvmlSystemGetDriverVersion.return_value = "535.0" + with patch( + "codecarbon.core.telemetry.collect.is_nvidia_system", + return_value=True, + ): + with patch.dict(sys.modules, {"pynvml": mock_pynvml}): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["gpu_memory_total_gb"], 8.0) + self.assertEqual(payload["cuda_version"], "12.4") + self.assertEqual(payload["gpu_driver_version"], "535.0") + + def test_extensive_payload_detects_docker_and_jupyter(self): + ctx = _tracker_context() + with patch("os.path.exists", return_value=True): + with patch( + "codecarbon.core.telemetry.collect._detect_notebook_environment", + return_value="jupyter", + ): + payload = build_payload(ctx, level=TelemetryLevel.extensive) + self.assertTrue(payload["in_container"]) + self.assertEqual(payload["container_runtime"], "docker") + self.assertEqual(payload["notebook_environment"], "jupyter") + + def test_minimal_payload_detects_virtualenv(self): + ctx = _tracker_context() + with patch.dict(os.environ, {"VIRTUAL_ENV": "/venv"}, clear=False): + with patch( + "codecarbon.core.telemetry.collect.sys.prefix", + "/venv", + create=True, + ): + with patch( + "codecarbon.core.telemetry.collect.sys.base_prefix", + "/usr", + create=True, + ): + payload = build_payload(ctx, level=TelemetryLevel.minimal) + self.assertEqual(payload["python_env_type"], "venv") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_config.py b/tests/test_telemetry_config.py new file mode 100644 index 000000000..e637cb243 --- /dev/null +++ b/tests/test_telemetry_config.py @@ -0,0 +1,302 @@ +"""Integration tests for telemetry tier resolution and config contract (Task 5).""" + +import os +import sys +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import MagicMock, patch + +from codecarbon.core.config import get_config_file_settings +from codecarbon.core.telemetry import ( + Telemetry, + TelemetryLevel, + TelemetrySettings, + post_private, +) +from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker +from tests.testutils import ensure_telemetry_run_duration, get_custom_mock_open + +if sys.platform == "darwin": + mock_platform_cli_setup = patch( + "codecarbon.core.powermetrics.ApplePowermetrics._setup_cli" + ) +else: + mock_platform_cli_setup = patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") + + +def _conf(level: str) -> str: + return f"[codecarbon]\ntelemetry_level = {level}\n" + + +class TestTelemetryConfigContract(unittest.TestCase): + def setUp(self) -> None: + Telemetry._default_warning_shown = False + + def tearDown(self) -> None: + Telemetry._default_warning_shown = False + + def test_warns_once_when_telemetry_not_explicit(self): + settings = TelemetrySettings.resolve(config_file_conf={}) + telemetry = Telemetry(settings) + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + telemetry.warn_if_implicit() + telemetry.warn_if_implicit() + self.assertEqual(mock_warning.call_count, 1) + self.assertIn("Minimal telemetry", mock_warning.call_args[0][0]) + + def test_no_warn_when_config_explicit(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + telemetry = Telemetry(settings) + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + telemetry.warn_if_implicit() + mock_warning.assert_not_called() + + def test_tier1_posts_to_telemetry_endpoint(self): + tier1_payload = { + "timestamp": datetime(2020, 1, 1, tzinfo=timezone.utc), + "telemetry_level": "minimal", + "os": "Linux", + "cpu_count": 4, + } + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://tier1.example"} + ) + with patch("codecarbon.core.telemetry.client.requests.post") as mock_post: + mock_post.return_value.status_code = 201 + mock_post.return_value.json.return_value = "telemetry-id" + post_private(settings, tier1_payload) + mock_post.assert_called_once() + self.assertEqual( + mock_post.call_args.kwargs["url"], "http://tier1.example/telemetry" + ) + self.assertEqual( + mock_post.call_args.kwargs["json"]["telemetry_level"], + TelemetryLevel.minimal.value, + ) + self.assertEqual(mock_post.call_args.kwargs["json"]["cpu_count"], 4) + + def test_legacy_env_codecarbon_telemetry_does_not_change_tier(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY": "disabled"}, + clear=False, + ): + from codecarbon.core.config import get_hierarchical_config + + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) + + def test_env_codecarbon_telemetry_level_overrides_file(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = Path(tmp) / ".codecarbon.config" + local_path.write_text(_conf("minimal")) + with patch( + "codecarbon.core.config._config_file_paths", + return_value=("/nonexistent/global", str(local_path)), + ): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, + clear=False, + ): + from codecarbon.core.config import get_hierarchical_config + + settings = TelemetrySettings.resolve( + config_file_conf=get_config_file_settings(), + external_conf=get_hierarchical_config(), + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_telemetry_api_url_env_used_for_tier2_client(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env-telemetry.example"}, + clear=False, + ): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env-telemetry.example") + + def test_telemetry_api_url_from_config_overrides_default(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://config-telemetry.example"} + ) + self.assertEqual(settings.api_url, "http://config-telemetry.example") + + +@mock_platform_cli_setup +class TestTrackerTelemetryFromConfig(unittest.TestCase): + def setUp(self) -> None: + Telemetry._default_warning_shown = False + self._config_patcher = None + + def tearDown(self) -> None: + if self._config_patcher: + self._config_patcher.stop() + Telemetry._default_warning_shown = False + + def _mock_config(self, conf: str) -> None: + self._config_patcher = patch( + "builtins.open", new_callable=get_custom_mock_open(conf, conf) + ) + self._config_patcher.start() + + def test_disabled_no_telemetry_on_stop(self, mock_cli_setup): + self._mock_config(_conf("disabled")) + with patch("codecarbon.core.telemetry.dispatcher.post_private") as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_not_called() + + def test_minimal_posts_tier1_on_stop_not_on_init(self, mock_cli_setup): + self._mock_config(_conf("minimal")) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + self.assertEqual( + mock_post.call_args[0][1]["telemetry_level"], + TelemetryLevel.minimal.value, + ) + + def test_tier2_posts_tier1_and_emission_on_stop_not_on_init(self, mock_cli_setup): + self._mock_config(_conf("extensive")) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + with patch( + "codecarbon.core.telemetry.client.ApiClient" + ) as mock_api_cls: + mock_api = MagicMock() + mock_api.add_emission.return_value = True + mock_api_cls.return_value = mock_api + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + measure_power_secs=1, + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + mock_api_cls.assert_called_once() + mock_api.add_emission.assert_called_once() + + def test_offline_minimal_posts_tier1_on_stop(self, mock_cli_setup): + self._mock_config(_conf("minimal")) + with ensure_telemetry_run_duration(): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", return_value=True + ) as mock_post: + tracker = OfflineEmissionsTracker( + country_iso_code="CAN", + save_to_api=False, + save_to_file=False, + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + + def test_warns_when_config_has_no_explicit_telemetry_level(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + env_without_telemetry = { + key: value + for key, value in os.environ.items() + if key.lower() + not in ( + "codecarbon_telemetry", + "codecarbon_telemetry_level", + ) + } + with patch.dict(os.environ, env_without_telemetry, clear=True): + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, + ): + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker(save_to_api=False, save_to_file=False) + configure_warnings = [ + c + for c in mock_warning.call_args_list + if c[0] and "telemetry" in str(c[0][0]).lower() + ] + self.assertEqual(len(configure_warnings), 1) + + def test_no_configure_warn_when_telemetry_level_kwarg_set(self, mock_cli_setup): + self._mock_config("[codecarbon]\n") + with patch( + "codecarbon.core.telemetry.dispatcher.logger.warning" + ) as mock_warning: + with patch("codecarbon.core.telemetry.dispatcher.post_private"): + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + EmissionsTracker( + telemetry_level="disabled", + save_to_api=False, + save_to_file=False, + ) + configure_warnings = [ + c + for c in mock_warning.call_args_list + if c[0] and "Minimal telemetry" in str(c[0][0]) + ] + self.assertEqual(len(configure_warnings), 0) + + def test_env_telemetry_disabled_does_not_change_resolved_level( + self, mock_cli_setup + ): + self._mock_config(_conf("minimal")) + with ensure_telemetry_run_duration(): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False + ): + with patch( + "codecarbon.core.telemetry.dispatcher.post_private", + return_value=True, + ) as mock_post: + with patch("codecarbon.external.geography.GeoMetadata.from_geo_js"): + tracker = EmissionsTracker( + save_to_api=False, save_to_file=False + ) + tracker.start() + tracker.stop() + mock_post.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_schemas.py b/tests/test_telemetry_schemas.py new file mode 100644 index 000000000..085649cee --- /dev/null +++ b/tests/test_telemetry_schemas.py @@ -0,0 +1,39 @@ +import unittest +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError + +from codecarbon.core.telemetry.schemas import TelemetryCreate, TelemetryLevel + + +class TestTelemetrySchemaValidation(unittest.TestCase): + def _minimal_payload(self) -> dict: + return { + "timestamp": datetime(2026, 1, 1, tzinfo=timezone.utc), + "telemetry_level": TelemetryLevel.minimal, + "os": "Linux", + } + + def test_rejects_disabled_telemetry_level(self): + with pytest.raises( + ValidationError, match="Disabled telemetry must not be submitted" + ): + TelemetryCreate( + **{**self._minimal_payload(), "telemetry_level": "disabled"} + ) + + def test_rejects_minimal_with_extensive_field(self): + with pytest.raises( + ValidationError, match="Minimal telemetry cannot include extensive fields" + ): + TelemetryCreate( + **{ + **self._minimal_payload(), + "total_emissions_kg": 0.5, + } + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_telemetry_settings.py b/tests/test_telemetry_settings.py new file mode 100644 index 000000000..e9a1e09f2 --- /dev/null +++ b/tests/test_telemetry_settings.py @@ -0,0 +1,174 @@ +import os +import unittest +from unittest.mock import patch + +from codecarbon.core.telemetry import ( + DEFAULT_TELEMETRY_API_KEY, + DEFAULT_TELEMETRY_API_URL, + DEFAULT_TELEMETRY_EXPERIMENT_ID, + TelemetryLevel, + TelemetrySettings, + parse_telemetry_level, +) + + +class TestParseTelemetryLevel(unittest.TestCase): + def test_parse_accepts_enum(self): + self.assertEqual( + parse_telemetry_level(TelemetryLevel.minimal), TelemetryLevel.minimal + ) + + def test_parse_normalizes_case(self): + self.assertEqual(parse_telemetry_level("EXTENSIVE"), TelemetryLevel.extensive) + + def test_parse_rejects_invalid(self): + with self.assertRaises(ValueError): + parse_telemetry_level("bogus") + + +class TestTelemetrySettingsResolve(unittest.TestCase): + def test_default_is_minimal_when_unset(self): + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertEqual(settings.level, TelemetryLevel.minimal) + self.assertEqual(settings.source, "default") + + def test_telemetry_level_from_config_file(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "disabled"} + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + self.assertEqual(settings.source, "file") + + def test_telemetry_level_extensive(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "extensive"} + ) + self.assertEqual(settings.level, TelemetryLevel.extensive) + + def test_env_telemetry_key_ignored(self): + with patch.dict(os.environ, {"CODECARBON_TELEMETRY": "disabled"}, clear=False): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry": "extensive"} + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) + + def test_invalid_level_falls_back_to_minimal(self): + with patch("codecarbon.core.telemetry.settings.logger.error") as mock_error: + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "bogus"} + ) + self.assertEqual(settings.level, TelemetryLevel.minimal) + mock_error.assert_called_once() + + def test_override_kwarg_takes_precedence_over_config_file(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, + override="disabled", + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_override_kwarg_takes_precedence_over_external_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_level": "extensive"}, + override="disabled", + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_external_conf_env_overrides_file_when_merged(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"}, + external_conf={"telemetry_level": "disabled"}, + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_env_telemetry_level_via_external_conf(self): + with patch.dict( + os.environ, {"CODECARBON_TELEMETRY_LEVEL": "disabled"}, clear=False + ): + from codecarbon.core.config import get_hierarchical_config + + settings = TelemetrySettings.resolve( + external_conf=get_hierarchical_config() + ) + self.assertEqual(settings.level, TelemetryLevel.disabled) + + def test_is_explicit_with_config_file(self): + settings = TelemetrySettings.resolve( + config_file_conf={"telemetry_level": "minimal"} + ) + self.assertTrue(settings.is_explicit) + + def test_is_explicit_with_override(self): + settings = TelemetrySettings.resolve(override="disabled") + self.assertTrue(settings.is_explicit) + + def test_is_explicit_with_env_telemetry_level(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_level": "minimal"} + ) + self.assertTrue(settings.is_explicit) + + def test_is_not_explicit_when_unset(self): + settings = TelemetrySettings.resolve(config_file_conf={}) + self.assertFalse(settings.is_explicit) + + +class TestTelemetryApiSettings(unittest.TestCase): + def test_api_url_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_url": "http://test.example"} + ) + self.assertEqual(settings.api_url, "http://test.example") + + def test_api_url_default(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_URL" + } + with patch.dict(os.environ, env, clear=True): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, DEFAULT_TELEMETRY_API_URL) + + def test_api_url_env_fallback(self): + with patch.dict( + os.environ, + {"CODECARBON_TELEMETRY_API_URL": "http://env.example"}, + clear=False, + ): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_url, "http://env.example") + + def test_api_key_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={"telemetry_api_key": "cpt_test"} + ) + self.assertEqual(settings.api_key, "cpt_test") + + def test_api_key_uses_public_default_when_unset(self): + env = { + k: v for k, v in os.environ.items() if k != "CODECARBON_TELEMETRY_API_KEY" + } + with patch.dict(os.environ, env, clear=True): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.api_key, DEFAULT_TELEMETRY_API_KEY) + + def test_experiment_id_from_conf(self): + settings = TelemetrySettings.resolve( + external_conf={ + "telemetry_experiment_id": "00000000-0000-0000-0000-000000000001" + } + ) + self.assertEqual(settings.experiment_id, "00000000-0000-0000-0000-000000000001") + + def test_experiment_id_uses_public_default_when_unset(self): + env = { + k: v + for k, v in os.environ.items() + if k != "CODECARBON_TELEMETRY_EXPERIMENT_ID" + } + with patch.dict(os.environ, env, clear=True): + settings = TelemetrySettings.resolve() + self.assertEqual(settings.experiment_id, DEFAULT_TELEMETRY_EXPERIMENT_ID) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/testutils.py b/tests/testutils.py index e3d1dc2d1..57d94fa46 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -1,6 +1,8 @@ import builtins import unittest +from contextlib import contextmanager from pathlib import Path +from unittest.mock import patch from codecarbon.input import DataSource @@ -33,3 +35,22 @@ def conditional_open_func(path, *args, **kwargs): return conditional_open_func return mocked_open + + +@contextmanager +def ensure_telemetry_run_duration(min_seconds: float = 10.0): + """Force tracker stop emissions duration above telemetry's 1s minimum.""" + from codecarbon.emissions_tracker import BaseEmissionsTracker + + original_prepare = BaseEmissionsTracker._prepare_emissions_data + + def prepare_with_min_duration(self): + data = original_prepare(self) + if data is not None and (data.duration is None or data.duration < min_seconds): + data.duration = min_seconds + return data + + with patch.object( + BaseEmissionsTracker, "_prepare_emissions_data", prepare_with_min_duration + ): + yield