diff --git a/codecarbon/__init__.py b/codecarbon/__init__.py index 15fc25cd0..9061aafb0 100644 --- a/codecarbon/__init__.py +++ b/codecarbon/__init__.py @@ -8,7 +8,7 @@ OfflineEmissionsTracker, track_emissions, ) -from .output import OutputMethod +from .output_methods.base_output import OutputMethod __all__ = [ "EmissionsTracker", diff --git a/codecarbon/cli/main.py b/codecarbon/cli/main.py index fd6545a3f..c10b32338 100644 --- a/codecarbon/cli/main.py +++ b/codecarbon/cli/main.py @@ -5,15 +5,12 @@ from pathlib import Path from typing import Optional -import questionary -import requests import typer from rich import print from rich.prompt import Confirm from typing_extensions import Annotated from codecarbon import __app_name__, __version__ -from codecarbon.cli.auth import authorize, get_access_token from codecarbon.cli.cli_utils import ( create_new_config_file, get_api_endpoint, @@ -21,10 +18,6 @@ get_existing_exp_id, overwrite_local_config, ) -from codecarbon.cli.monitor import run_and_monitor -from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone -from codecarbon.core.schemas import ExperimentCreate, OrganizationCreate, ProjectCreate -from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker API_URL = os.environ.get("API_URL", "https://dashboard.codecarbon.io/api") @@ -68,6 +61,9 @@ def version( def show_config(path: Path = Path("./.codecarbon.config")) -> None: + from codecarbon.cli.auth import get_access_token + from codecarbon.core.api_client import ApiClient + d = get_config(path) print("Current configuration : \n") print("Config file content : ") @@ -114,6 +110,9 @@ def api_get(): """ ex: test-api """ + from codecarbon.cli.auth import get_access_token + from codecarbon.core.api_client import ApiClient + api_endpoint = get_api_endpoint() api = ApiClient(endpoint_url=api_endpoint) api.set_access_token(get_access_token()) @@ -123,6 +122,9 @@ def api_get(): @codecarbon.command("login", short_help="Login to CodeCarbon") def login(): + from codecarbon.cli.auth import authorize, get_access_token + from codecarbon.core.api_client import ApiClient + authorize() api_endpoint = get_api_endpoint() api = ApiClient(endpoint_url=api_endpoint) @@ -132,6 +134,10 @@ def login(): def get_api_key(project_id: str): + import requests + + from codecarbon.cli.auth import get_access_token + api_endpoint = get_api_endpoint() api_endpoint = api_endpoint.rstrip("/") req = requests.post( @@ -161,6 +167,13 @@ def config(): """ Initialize CodeCarbon, this will prompt you for configuration of Organisation/Team/Project/Experiment. """ + from codecarbon.cli.auth import get_access_token + from codecarbon.core.api_client import ApiClient, get_datetime_with_timezone + from codecarbon.core.schemas import ( + ExperimentCreate, + OrganizationCreate, + ProjectCreate, + ) print("Welcome to CodeCarbon configuration wizard") home = Path.home() @@ -342,6 +355,10 @@ def monitor( str, typer.Option(help="Region/province for offline mode"), ] = None, + log_level: Annotated[ + str, + typer.Option(help="Log level (critical, error, warning, info, debug)"), + ] = "error", ): """Monitor your machine's carbon emissions.""" @@ -349,6 +366,7 @@ def monitor( tracker_args = { "measure_power_secs": measure_power_secs, "api_call_interval": api_call_interval, + "log_level": log_level, } # Set up the tracker arguments based on mode (offline vs online) and validate required args for each mode if offline: @@ -375,8 +393,12 @@ def monitor( tracker_args = {**tracker_args, "save_to_api": api} + from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker + # If extra args are provided (e.g. `codecarbon monitor -- my_script.py`), delegate to `run_and_monitor` if getattr(ctx, "args", None): + from codecarbon.cli.monitor import run_and_monitor + return run_and_monitor(ctx, offline=offline, **tracker_args) # Instantiate the tracker @@ -417,6 +439,8 @@ def detect(): """ Detects hardware and prints information without running any measurements. """ + from codecarbon.emissions_tracker import EmissionsTracker + print("Detecting hardware...") tracker = EmissionsTracker(save_to_file=False) hardware_info = tracker.get_detected_hardware() @@ -438,6 +462,8 @@ def detect(): def questionary_prompt(prompt, list_options, default): + import questionary + value = questionary.select( prompt, list_options, diff --git a/codecarbon/cli/monitor.py b/codecarbon/cli/monitor.py index 98fa4e244..41b3ca353 100644 --- a/codecarbon/cli/monitor.py +++ b/codecarbon/cli/monitor.py @@ -8,8 +8,6 @@ from rich import print from typing_extensions import Annotated -from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker - def run_and_monitor( ctx: typer.Context, @@ -50,12 +48,15 @@ def run_and_monitor( directory. The file path is shown in the final report. """ # Suppress all CodeCarbon logs during execution + from codecarbon.emissions_tracker import EmissionsTracker, OfflineEmissionsTracker from codecarbon.external.logger import set_logger_level set_logger_level(log_level) - # Get the command from remaining args - command = ctx.args + # Get the command from remaining args (strip nested subcommand / `--` leftovers) + command = list(getattr(ctx, "args", None) or []) + while command and command[0] in ("monitor", "--"): + command.pop(0) if not command: print( diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index 9349b9dc1..58f4932cb 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -10,7 +10,6 @@ import json from datetime import timedelta, tzinfo -import arrow import requests from codecarbon.core.schemas import ( @@ -22,12 +21,11 @@ ) from codecarbon.external.logger import logger -# from codecarbon.output import EmissionsData - def get_datetime_with_timezone(): - timestamp = str(arrow.now().isoformat()) - return timestamp + import arrow + + return str(arrow.now().isoformat()) class ApiClient: # (AsyncClient) diff --git a/codecarbon/core/cpu.py b/codecarbon/core/cpu.py index 0f4ebfdb6..28af4095c 100644 --- a/codecarbon/core/cpu.py +++ b/codecarbon/core/cpu.py @@ -4,14 +4,16 @@ https://software.intel.com/content/www/us/en/develop/articles/intel-power-gadget.html """ +from __future__ import annotations + import os import re import shutil import subprocess import sys +from functools import lru_cache from typing import Dict, Optional, Tuple -import pandas as pd import psutil from rapidfuzz import fuzz, process, utils @@ -19,12 +21,12 @@ from codecarbon.core.units import Time from codecarbon.core.util import count_cpus, detect_cpu_model from codecarbon.external.logger import logger -from codecarbon.input import DataSource # default W value per core for a CPU if no model is found in the ref csv DEFAULT_POWER_PER_CORE = 4 +@lru_cache(maxsize=1) def is_powergadget_available() -> bool: """ Checks if Intel Power Gadget is available on the system. @@ -44,6 +46,10 @@ def is_powergadget_available() -> bool: return False +def clear_powergadget_cache() -> None: + is_powergadget_available.cache_clear() + + def _get_candidate_bases(rapl_dir: str) -> list: """Get list of directories to scan for RAPL files.""" default_rapl_dir = "/sys/class/powercap/intel-rapl/subsystem" @@ -366,6 +372,8 @@ def get_cpu_details(self) -> Dict: self._log_values() cpu_details = {} try: + import pandas as pd + cpu_data = pd.read_csv(self._log_file_path).dropna() for col_name in cpu_data.columns: if col_name in ["System Time", "Elapsed Time (sec)", "RDTSC"]: @@ -887,11 +895,13 @@ def __init__(self): self.model, self.tdp = self._main() @staticmethod - def _get_cpu_constant_power(match: str, cpu_power_df: pd.DataFrame) -> int: + def _get_cpu_constant_power(match: str, cpu_power_df) -> int: """Extract constant power from matched CPU""" return float(cpu_power_df[cpu_power_df["Name"] == match]["TDP"].values[0]) def _get_cpu_power_from_registry(self, cpu_model_raw: str) -> Optional[int]: + from codecarbon.input import DataSource + cpu_power_df = DataSource().get_cpu_power_data() cpu_matching = self._get_matching_cpu(cpu_model_raw, cpu_power_df) if cpu_matching: @@ -899,9 +909,7 @@ def _get_cpu_power_from_registry(self, cpu_model_raw: str) -> Optional[int]: return power return None - def _get_matching_cpu( - self, model_raw: str, cpu_df: pd.DataFrame, greedy=False - ) -> str: + def _get_matching_cpu(self, model_raw: str, cpu_df, greedy=False) -> str: """ Get matching cpu name diff --git a/codecarbon/core/emissions.py b/codecarbon/core/emissions.py index 953ca47b3..6b173fd96 100644 --- a/codecarbon/core/emissions.py +++ b/codecarbon/core/emissions.py @@ -8,8 +8,6 @@ from typing import Dict, Optional -import pandas as pd - from codecarbon.core import electricitymaps_api from codecarbon.core.units import EmissionsPerKWh, Energy from codecarbon.external.geography import CloudMetadata, GeoMetadata @@ -64,7 +62,7 @@ def get_cloud_emissions( ) return energy.kWh * (self._force_carbon_intensity_g_co2e_kwh / 1000.0) - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() + df = self._data_source.get_cloud_emissions_data() try: emissions_per_kWh: EmissionsPerKWh = EmissionsPerKWh.from_g_per_kWh( df.loc[ @@ -99,7 +97,7 @@ def get_cloud_country_name(self, cloud: CloudMetadata) -> str: """ Returns the Country Name where the cloud region is located """ - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() + df = self._data_source.get_cloud_emissions_data() flags = (df["provider"] == cloud.provider) & (df["region"] == cloud.region) selected = df.loc[flags] if not len(selected): @@ -114,7 +112,7 @@ def get_cloud_country_iso_code(self, cloud: CloudMetadata) -> str: """ Returns the Country ISO Code where the cloud region is located """ - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() + df = self._data_source.get_cloud_emissions_data() flags = (df["provider"] == cloud.provider) & (df["region"] == cloud.region) selected = df.loc[flags] if not len(selected): @@ -129,7 +127,7 @@ def get_cloud_geo_region(self, cloud: CloudMetadata) -> str: """ Returns the State/City where the cloud region is located """ - df: pd.DataFrame = self._data_source.get_cloud_emissions_data() + df = self._data_source.get_cloud_emissions_data() flags = (df["provider"] == cloud.provider) & (df["region"] == cloud.region) selected = df.loc[flags] if not len(selected): diff --git a/codecarbon/core/gpu_amd.py b/codecarbon/core/gpu_amd.py index bd8eeb226..e79e9f43c 100644 --- a/codecarbon/core/gpu_amd.py +++ b/codecarbon/core/gpu_amd.py @@ -1,21 +1,26 @@ import subprocess from collections import namedtuple +from functools import lru_cache from typing import Callable from codecarbon.core.gpu_device import GPUDevice from codecarbon.external.logger import logger +@lru_cache(maxsize=1) def is_rocm_system(): """Returns True if the system has an rocm-smi interface.""" try: - # Check if rocm-smi is available subprocess.check_output(["rocm-smi", "--help"]) return True except (subprocess.CalledProcessError, OSError): return False +def clear_rocm_system_cache() -> None: + is_rocm_system.cache_clear() + + try: import amdsmi diff --git a/codecarbon/core/gpu_nvidia.py b/codecarbon/core/gpu_nvidia.py index ddda4c57d..bf7cb2909 100644 --- a/codecarbon/core/gpu_nvidia.py +++ b/codecarbon/core/gpu_nvidia.py @@ -1,21 +1,26 @@ import subprocess from dataclasses import dataclass +from functools import lru_cache from typing import Any, Union from codecarbon.core.gpu_device import GPUDevice from codecarbon.external.logger import logger +@lru_cache(maxsize=1) def is_nvidia_system(): """Returns True if the system has an nvidia-smi interface.""" try: - # Check if nvidia-smi is available subprocess.check_output(["nvidia-smi", "--help"]) return True except Exception: return False +def clear_nvidia_system_cache() -> None: + is_nvidia_system.cache_clear() + + try: import pynvml diff --git a/codecarbon/core/hardware_cache.py b/codecarbon/core/hardware_cache.py new file mode 100644 index 000000000..45347aa5d --- /dev/null +++ b/codecarbon/core/hardware_cache.py @@ -0,0 +1,236 @@ +""" +Process-level cache for hardware detection and setup. + +Reuses the outcome of the first tracker hardware probe so additional runs on +the same device (same process) skip repeated powermetrics, cpuinfo, and GPU +detection work. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +from codecarbon.core.config import normalize_gpu_ids + +if TYPE_CHECKING: + from codecarbon.core.resource_tracker import ResourceTracker + +DEFAULT_RAPL_DIR = "/sys/class/powercap/intel-rapl/subsystem" + +CONF_KEYS = ( + "ram_total_size", + "cpu_count", + "cpu_physical_count", + "cpu_model", + "gpu_count", + "gpu_model", + "gpu_ids", +) + +_cache_lock = threading.Lock() +_plans: Dict["_HardwareCacheKey", "_HardwarePlan"] = {} +_tdp = None + + +@dataclass(frozen=True) +class _HardwareCacheKey: + tracking_mode: str + force_cpu_power: Any + force_ram_power: Any + force_mode_cpu_load: bool + gpu_ids: Any + rapl_include_dram: bool + rapl_prefer_psys: bool + + +@dataclass +class _HardwarePlan: + ram_tracker: str + cpu_tracker: str + gpu_tracker: str + conf: Dict[str, Any] = field(default_factory=dict) + hardware_specs: List[Dict[str, Any]] = field(default_factory=list) + + +def _canonical_gpu_ids( + gpu_ids: Optional[List], +) -> Optional[Tuple[str, ...]]: + """Normalize GPU ids to a stable cache-key form (tuple of strings).""" + if gpu_ids is None: + return None + if not isinstance(gpu_ids, (list, tuple)): + gpu_ids = [gpu_ids] + normalized = normalize_gpu_ids(list(gpu_ids)) + if not normalized: + return None + return tuple(str(gpu_id) for gpu_id in normalized) + + +def make_key(tracker) -> _HardwareCacheKey: + return _HardwareCacheKey( + tracking_mode=tracker._tracking_mode, + force_cpu_power=tracker._force_cpu_power, + force_ram_power=tracker._force_ram_power, + force_mode_cpu_load=bool(tracker._conf.get("force_mode_cpu_load", False)), + gpu_ids=_canonical_gpu_ids(tracker._gpu_ids), + rapl_include_dram=bool(tracker._rapl_include_dram), + rapl_prefer_psys=bool(tracker._rapl_prefer_psys), + ) + + +def get_cached_tdp(cpu_module): + """Return a shared cpu.TDP() instance for this process.""" + global _tdp + if _tdp is None: + _tdp = cpu_module.TDP() + return _tdp + + +def _hardware_kind(hw) -> str: + """Classify hardware without isinstance (safe if modules were reloaded).""" + name = type(hw).__name__ + if name == "RAM": + return "ram" + if name == "CPU": + return "cpu" + if name == "AppleSiliconChip": + return "apple_chip" + if name == "GPU": + return "gpu" + raise TypeError(f"Unsupported hardware type for cache: {type(hw)}") + + +def _spec_from_hardware(hw) -> Dict[str, Any]: + kind = _hardware_kind(hw) + if kind == "ram": + return { + "kind": "ram", + "tracking_mode": hw._tracking_mode, + "force_ram_power": hw._force_ram_power, + } + if kind == "cpu": + spec: Dict[str, Any] = { + "kind": "cpu", + "mode": hw._mode, + "model": hw._model, + "tdp": hw._tdp, + "tracking_mode": hw._tracking_mode, + "rapl_include_dram": False, + "rapl_prefer_psys": False, + } + if hw._mode == "intel_rapl" and hasattr(hw, "_intel_interface"): + intel = hw._intel_interface + spec["rapl_include_dram"] = getattr(intel, "rapl_include_dram", False) + spec["rapl_prefer_psys"] = getattr(intel, "rapl_prefer_psys", False) + spec["rapl_dir"] = getattr(intel, "_lin_rapl_dir", DEFAULT_RAPL_DIR) + return spec + if kind == "apple_chip": + return { + "kind": "apple_chip", + "model": hw._model, + "chip_part": hw.chip_part, + } + if kind == "gpu": + gpu_ids = _canonical_gpu_ids(hw.gpu_ids) + return {"kind": "gpu", "gpu_ids": list(gpu_ids) if gpu_ids else None} + raise TypeError(f"Unsupported hardware type for cache: {type(hw)}") + + +def _hardware_from_spec(spec: Dict[str, Any], output_dir: str): + from codecarbon.external.hardware import CPU, GPU, AppleSiliconChip + from codecarbon.external.ram import RAM + + kind = spec["kind"] + if kind == "ram": + return RAM( + tracking_mode=spec["tracking_mode"], + force_ram_power=spec.get("force_ram_power"), + ) + if kind == "cpu": + return CPU( + output_dir=output_dir, + mode=spec["mode"], + model=spec["model"], + tdp=spec["tdp"], + tracking_mode=spec["tracking_mode"], + rapl_dir=spec.get("rapl_dir", DEFAULT_RAPL_DIR), + rapl_include_dram=spec.get("rapl_include_dram", False), + rapl_prefer_psys=spec.get("rapl_prefer_psys", False), + ) + if kind == "apple_chip": + return AppleSiliconChip( + output_dir=output_dir, + model=spec["model"], + chip_part=spec["chip_part"], + ) + if kind == "gpu": + gpu_ids = _canonical_gpu_ids(spec.get("gpu_ids")) + return GPU.from_utils(gpu_ids=list(gpu_ids) if gpu_ids else None) + raise ValueError(f"Unknown hardware spec kind: {kind}") + + +def capture(resource_tracker: "ResourceTracker") -> _HardwarePlan: + tracker = resource_tracker.tracker + conf = {k: tracker._conf[k] for k in CONF_KEYS if k in tracker._conf} + return _HardwarePlan( + ram_tracker=resource_tracker.ram_tracker, + cpu_tracker=resource_tracker.cpu_tracker, + gpu_tracker=resource_tracker.gpu_tracker, + conf=conf, + hardware_specs=[_spec_from_hardware(hw) for hw in tracker._hardware], + ) + + +def apply(resource_tracker: "ResourceTracker", plan: _HardwarePlan) -> None: + tracker = resource_tracker.tracker + resource_tracker.ram_tracker = plan.ram_tracker + resource_tracker.cpu_tracker = plan.cpu_tracker + resource_tracker.gpu_tracker = plan.gpu_tracker + tracker._conf.update(plan.conf) + if "gpu_ids" in plan.conf: + tracker._gpu_ids = plan.conf["gpu_ids"] + tracker._hardware = [ + _hardware_from_spec(spec, tracker._output_dir) for spec in plan.hardware_specs + ] + + +def get_or_run_setup( + resource_tracker: "ResourceTracker", + setup_fn, +) -> None: + """Apply cached hardware plan or run full setup once per cache key.""" + key = make_key(resource_tracker.tracker) + with _cache_lock: + plan = _plans.get(key) + if plan is not None: + apply(resource_tracker, plan) + return + setup_fn() + _plans[key] = capture(resource_tracker) + + +def clear_cache() -> None: + """Clear cached plans (for tests).""" + global _tdp + import sys + + with _cache_lock: + _plans.clear() + _tdp = None + + for mod_name, clear_fn in ( + ("codecarbon.core.gpu_nvidia", "clear_nvidia_system_cache"), + ("codecarbon.core.gpu_amd", "clear_rocm_system_cache"), + ("codecarbon.core.cpu", "clear_powergadget_cache"), + ("codecarbon.core.powermetrics", "clear_powermetrics_cache"), + ): + mod = sys.modules.get(mod_name) + if mod is not None: + getattr(mod, clear_fn)() + + if "codecarbon.external.hardware" in sys.modules: + from codecarbon.external.hardware import clear_cpu_load_prime_cache + + clear_cpu_load_prime_cache() diff --git a/codecarbon/core/powermetrics.py b/codecarbon/core/powermetrics.py index c445fc918..b59995154 100644 --- a/codecarbon/core/powermetrics.py +++ b/codecarbon/core/powermetrics.py @@ -3,6 +3,8 @@ import shutil import subprocess import sys +import time +from functools import lru_cache from typing import Dict import numpy as np @@ -11,11 +13,11 @@ from codecarbon.external.logger import logger +@lru_cache(maxsize=1) def is_powermetrics_available() -> bool: try: ApplePowermetrics() - response = _has_powermetrics_sudo() - return response + return _has_powermetrics_sudo() except Exception as e: logger.debug( "Not using PowerMetrics, an exception occurred while instantiating" @@ -24,6 +26,10 @@ def is_powermetrics_available() -> bool: return False +def clear_powermetrics_cache() -> None: + is_powermetrics_available.cache_clear() + + def _has_powermetrics_sudo() -> bool: if shutil.which("sudo") is None: logger.debug("sudo not available, we won't use Apple PowerMetrics.") @@ -51,6 +57,13 @@ def _has_powermetrics_sudo() -> bool: stderr=subprocess.PIPE, text=True, ) as process: + deadline = time.time() + 3 + while process.poll() is None and time.time() < deadline: + time.sleep(0.05) + if process.poll() is None: + process.kill() + logger.debug("PowerMetrics sudo check timed out.") + return False _, stderr = process.communicate() if re.search(r"[sudo].*password", stderr): diff --git a/codecarbon/core/resource_tracker.py b/codecarbon/core/resource_tracker.py index 67786189d..8a6496924 100644 --- a/codecarbon/core/resource_tracker.py +++ b/codecarbon/core/resource_tracker.py @@ -1,8 +1,10 @@ from collections import Counter +from concurrent.futures import ThreadPoolExecutor from typing import List, Union from codecarbon.core import cpu, gpu, powermetrics from codecarbon.core.config import normalize_gpu_ids +from codecarbon.core.hardware_cache import get_cached_tdp, get_or_run_setup from codecarbon.core.util import ( detect_cpu_model, is_linux_os, @@ -62,7 +64,11 @@ def _setup_power_gadget(self): """Set up CPU tracking using Intel Power Gadget.""" logger.info("Tracking Intel CPU via Power Gadget") self.cpu_tracker = "Power Gadget" - hardware_cpu = CPU.from_utils(self.tracker._output_dir, "intel_power_gadget") + hardware_cpu = CPU.from_utils( + self.tracker._output_dir, + "intel_power_gadget", + tracking_mode=self.tracker._tracking_mode, + ) self.tracker._hardware.append(hardware_cpu) self.tracker._conf["cpu_model"] = hardware_cpu.get_model() return True @@ -74,6 +80,7 @@ def _setup_rapl(self): hardware_cpu = CPU.from_utils( output_dir=self.tracker._output_dir, mode="intel_rapl", + tracking_mode=self.tracker._tracking_mode, rapl_include_dram=self.tracker._rapl_include_dram, rapl_prefer_psys=self.tracker._rapl_prefer_psys, ) @@ -117,6 +124,23 @@ def _get_install_instructions(self): return "Linux OS detected: Please ensure RAPL files exist, and are readable, at /sys/class/powercap/intel-rapl/subsystem to measure CPU" return "" + def _setup_cpu_load_fast(self, model: str) -> bool: + """Set up cpu_load mode without loading the TDP registry (faster cold start).""" + if not cpu.is_psutil_available(): + return False + logger.warning("No CPU tracking mode found. Falling back on CPU load mode.") + hardware_cpu = CPU.from_utils( + self.tracker._output_dir, + MODE_CPU_LOAD, + model or "Unknown CPU", + None, + tracking_mode=self.tracker._tracking_mode, + ) + self.cpu_tracker = MODE_CPU_LOAD + self.tracker._conf["cpu_model"] = hardware_cpu.get_model() + self.tracker._hardware.append(hardware_cpu) + return True + def _setup_fallback_tracking(self, tdp, max_power): """Set up fallback CPU tracking using TDP estimation.""" cpu_tracking_install_instructions = self._get_install_instructions() @@ -179,6 +203,30 @@ def _setup_fallback_tracking(self, tdp, max_power): hardware_cpu = CPU.from_utils(self.tracker._output_dir, "constant") self.tracker._hardware.append(hardware_cpu) + def _try_platform_cpu_backend(self) -> bool: + """Try platform-preferred CPU backends when force_cpu_power is unset.""" + if is_linux_os() and cpu.is_rapl_available(): + self._setup_rapl() + return True + if is_mac_os(): + cpu_model = detect_cpu_model() or "" + if is_mac_arm(cpu_model): + if self._setup_cpu_load_fast(cpu_model): + return True + if powermetrics.is_powermetrics_available(): + self._setup_powermetrics() + return True + elif cpu.is_powergadget_available(): + self._setup_power_gadget() + return True + elif powermetrics.is_powermetrics_available(): + self._setup_powermetrics() + return True + elif is_windows_os() and cpu.is_powergadget_available(): + self._setup_power_gadget() + return True + return False + def set_CPU_tracking(self): logger.info("[setup] CPU Tracking...") cpu_number = self.tracker._conf.get("cpu_physical_count") @@ -195,29 +243,21 @@ def set_CPU_tracking(self): # Try force CPU load mode if requested if self.tracker._conf.get("force_mode_cpu_load", False): if tdp is None: - tdp = cpu.TDP() + tdp = get_cached_tdp(cpu) if max_power is None: max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None if tdp.tdp is not None or self.tracker._force_cpu_power is not None: if self._setup_cpu_load_mode(tdp, max_power): return - # Try various tracking methods in order of preference - if cpu.is_powergadget_available() and self.tracker._force_cpu_power is None: - self._setup_power_gadget() - elif cpu.is_rapl_available() and self.tracker._force_cpu_power is None: - self._setup_rapl() - elif ( - powermetrics.is_powermetrics_available() - and self.tracker._force_cpu_power is None - ): - self._setup_powermetrics() - else: - if tdp is None: - tdp = cpu.TDP() - if max_power is None: - max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None - self._setup_fallback_tracking(tdp, max_power) + if self.tracker._force_cpu_power is None and self._try_platform_cpu_backend(): + return + + if tdp is None: + tdp = get_cached_tdp(cpu) + if max_power is None: + max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None + self._setup_fallback_tracking(tdp, max_power) def set_GPU_tracking(self): logger.info("[setup] GPU Tracking...") @@ -250,14 +290,13 @@ def set_GPU_tracking(self): self.tracker._conf.setdefault("gpu_count", 0) self.tracker._conf.setdefault("gpu_model", "") - def set_CPU_GPU_ram_tracking(self): - """ - Set up CPU, GPU and RAM tracking based on the user's configuration. - param tracker: BaseEmissionsTracker object - """ + def _run_full_hardware_setup(self) -> None: self.set_RAM_tracking() - self.set_CPU_tracking() - self.set_GPU_tracking() + with ThreadPoolExecutor(max_workers=2) as pool: + cpu_future = pool.submit(self.set_CPU_tracking) + gpu_future = pool.submit(self.set_GPU_tracking) + cpu_future.result() + gpu_future.result() logger.info( f"""The below tracking methods have been set up: @@ -266,3 +305,10 @@ def set_CPU_GPU_ram_tracking(self): GPU Tracking Method: {self.gpu_tracker} """ ) + + def set_CPU_GPU_ram_tracking(self): + """ + Set up CPU, GPU and RAM tracking based on the user's configuration. + param tracker: BaseEmissionsTracker object + """ + get_or_run_setup(self, self._run_full_hardware_setup) diff --git a/codecarbon/core/util.py b/codecarbon/core/util.py index 744b2e3e5..3bb0ca39c 100644 --- a/codecarbon/core/util.py +++ b/codecarbon/core/util.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Optional, Union -import cpuinfo import psutil from codecarbon.external.logger import logger @@ -76,6 +75,8 @@ def backup(file_path: Union[str, Path], ext: Optional[str] = ".bak") -> None: @lru_cache(maxsize=1) def detect_cpu_model() -> Optional[str]: + import cpuinfo + cpu_info = cpuinfo.get_cpu_info() if cpu_info: cpu_model_detected = cpu_info.get("brand_raw", "") diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 88da92628..efd3830ea 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -3,6 +3,8 @@ OfflineEmissionsTracker, context manager and decorator @track_emissions """ +from __future__ import annotations + import dataclasses import os import platform @@ -13,17 +15,14 @@ from abc import ABC, abstractmethod from datetime import datetime from functools import wraps -from typing import Any, Callable, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import psutil from codecarbon._version import __version__ from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids -from codecarbon.core.emissions import Emissions -from codecarbon.core.resource_tracker import ResourceTracker from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress -from codecarbon.external.geography import CloudMetadata, GeoMetadata from codecarbon.external.hardware import CPU, GPU, AppleSiliconChip from codecarbon.external.logger import logger, set_logger_format, set_logger_level from codecarbon.external.ram import RAM @@ -31,18 +30,12 @@ from codecarbon.external.task import Task from codecarbon.input import DataSource from codecarbon.lock import Lock -from codecarbon.output import ( - BaseOutput, - BoAmpsOutput, - CodeCarbonAPIOutput, - EmissionsData, - FileOutput, - HTTPOutput, - LogfireOutput, - LoggerOutput, - OutputMethod, - PrometheusOutput, -) +from codecarbon.output_methods.base_output import BaseOutput, OutputMethod +from codecarbon.output_methods.emissions_data import EmissionsData + +if TYPE_CHECKING: + from codecarbon.external.geography import CloudMetadata, GeoMetadata + from codecarbon.output_methods.logger import LoggerOutput # /!\ Warning: current implementation prevents the user from setting any value to None # from the script call @@ -304,6 +297,15 @@ def _initialize_runtime_state(self) -> None: self._active_task: Optional[str] = None self._active_task_emissions_at_start: Optional[EmissionsData] = None self._hardware = [] + self._hardware_initialized = False + + def _ensure_hardware_ready(self) -> None: + if self._hardware_initialized: + return + self._populate_system_metadata() + self._initialize_hardware_tracking() + self._hardware_initialized = True + self._log_tracker_metadata() def _populate_system_metadata(self) -> None: self._conf["os"] = platform.platform() @@ -312,6 +314,8 @@ def _populate_system_metadata(self) -> None: self._conf["cpu_physical_count"] = count_physical_cpus() def _initialize_hardware_tracking(self) -> None: + from codecarbon.core.resource_tracker import ResourceTracker + resource_tracker = ResourceTracker(self) resource_tracker.set_CPU_GPU_ram_tracking() self._conf["hardware"] = [item.description() for item in self._hardware] @@ -349,21 +353,38 @@ def _initialize_scheduler_state(self) -> None: def _initialize_emissions_context(self) -> None: self._data_source = DataSource() - cloud: CloudMetadata = self._get_cloud_metadata() - self._geo = self._get_geo_metadata() - - if cloud.is_on_private_infra: - self._conf["longitude"] = self._geo.longitude - self._conf["latitude"] = self._geo.latitude + self._geo = None + self._emissions = None + def _ensure_cloud_conf(self) -> None: + if self._conf.get("_cloud_conf_initialized"): + return + cloud = self._get_cloud_metadata() self._conf["region"] = cloud.region self._conf["provider"] = cloud.provider - self._emissions: Emissions = Emissions( + self._conf["_cloud_conf_initialized"] = True + + def _ensure_emissions_engine(self) -> None: + if self._emissions is not None: + return + from codecarbon.core.emissions import Emissions + + self._emissions = Emissions( self._data_source, self._electricitymaps_api_token, force_carbon_intensity_g_co2e_kwh=self.force_carbon_intensity_g_co2e_kwh, ) + def _ensure_geo_metadata(self) -> None: + """Load geo metadata on first use to avoid blocking tracker construction.""" + if self._geo is not None: + return + self._geo = self._get_geo_metadata() + cloud: CloudMetadata = self._get_cloud_metadata() + if cloud.is_on_private_infra: + self._conf["longitude"] = self._geo.longitude + self._conf["latitude"] = self._geo.latitude + def __init__( self, project_name: Optional[str] = _sentinel, @@ -578,9 +599,6 @@ def __init__( set_logger_level(self._log_level) set_logger_format(self._logger_preamble) self._initialize_runtime_state() - self._populate_system_metadata() - self._initialize_hardware_tracking() - self._log_tracker_metadata() self._initialize_scheduler_state() self._initialize_emissions_context() self._init_output_methods(api_key=self._api_key) @@ -591,6 +609,18 @@ def _init_output_methods(self, *, api_key: str = None): """ methods = set(self._output_methods) if self._output_methods else set() + if not methods and not self._emissions_endpoint: + self.run_id = uuid.uuid4() + return + + from codecarbon.output_methods.boamps import BoAmpsOutput + from codecarbon.output_methods.file import FileOutput + from codecarbon.output_methods.http import CodeCarbonAPIOutput, HTTPOutput + from codecarbon.output_methods.metrics.logfire import LogfireOutput + from codecarbon.output_methods.metrics.prometheus import PrometheusOutput + + methods = set(self._output_methods) if self._output_methods else set() + if OutputMethod.CSV in methods: self._output_handlers.append( FileOutput( @@ -641,6 +671,7 @@ def get_detected_hardware(self) -> Dict[str, Any]: Get the detected hardware. :return: A dictionary containing hardware data. """ + self._ensure_hardware_ready() hardware_info = { "ram_total_size": self._conf.get("ram_total_size"), "cpu_count": self._conf.get("cpu_count"), @@ -672,15 +703,11 @@ def start(self) -> None: "Another instance of codecarbon is already running. Exiting." ) return - try: - _ = self._emissions - except AttributeError: - logger.error("Tracker not initialized. Please check the logs.") - return if self._start_time is not None: logger.warning("Already started tracking") return + self._ensure_hardware_ready() self._last_measured_time = self._start_time = time.perf_counter() # Clear utilization history for fresh measurements @@ -694,7 +721,9 @@ def start(self) -> None: hardware.start() self._scheduler.start() - self._scheduler_monitor_power.start() + if self._output_handlers: + self._scheduler_monitor_power.start() + self._measure_power_and_energy() def start_task(self, task_name=None) -> None: """ @@ -712,11 +741,13 @@ def start_task(self, task_name=None) -> None: ) return try: - _ = self._emissions - except AttributeError: + self._ensure_emissions_engine() + except Exception: logger.error("Tracker not initialized. Please check the logs.") return + self._ensure_hardware_ready() + # Stop scheduler as we do not want it to interfere with the task measurement if self._scheduler: self._scheduler.stop() @@ -839,7 +870,7 @@ def flush(self) -> Optional[float]: # Run to calculate the power used from last # scheduled measurement to shutdown - self._measure_power_and_energy() + self._measure_power_and_energy_if_stale() emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) @@ -887,7 +918,7 @@ def stop(self) -> Optional[float]: # Run to calculate the power used from last # scheduled measurement to shutdown # or if scheduler interval was longer than the run - self._measure_power_and_energy() + self._measure_power_and_energy_if_stale() emissions_data = self._prepare_emissions_data() emissions_data_delta = self._compute_emissions_delta(emissions_data) @@ -926,6 +957,8 @@ def _update_emissions(self) -> None: Compute emissions for the energy consumed since the last update and add them to the total emissions. """ + self._ensure_geo_metadata() + self._ensure_emissions_engine() delta_energy = self._total_energy - self._last_energy_covered if delta_energy.kWh > 0: cloud: CloudMetadata = self._get_cloud_metadata() @@ -946,7 +979,8 @@ def _prepare_emissions_data(self) -> EmissionsData: :return: EmissionsData object with the total emissions data. """ self._update_emissions() - cloud: CloudMetadata = self._get_cloud_metadata() + self._ensure_cloud_conf() + cloud = self._get_cloud_metadata() duration: Time = Time.from_seconds(time.perf_counter() - self._start_time) emissions = self._total_emissions @@ -1204,6 +1238,11 @@ def _do_measurements(self) -> None: f"{self._total_energy.kWh:.6f} kWh of electricity and {self._total_water.litres:.6f} L of water were used since the beginning." ) + def _measure_power_and_energy_if_stale(self, min_interval_s: float = 0.05) -> None: + """Measure only if the last sample is older than ``min_interval_s``.""" + if time.perf_counter() - self._last_measured_time >= min_interval_s: + self._measure_power_and_energy() + def _measure_power_and_energy(self) -> None: """ A function that is periodically run by the `BackgroundScheduler` @@ -1317,40 +1356,48 @@ def __init__( "Cloud Region must be provided " + " if cloud provider is set" ) - df = DataSource().get_cloud_emissions_data() - if ( - len( - df.loc[ - (df["provider"] == self._cloud_provider) - & (df["region"] == self._cloud_region) - ] - ) - == 0 - ): - logger.error( - "Cloud Provider/Region " - f"{self._cloud_provider} {self._cloud_region} " - "not found in cloud emissions data." - ) - if self._country_iso_code: - try: - self._country_name: str = DataSource().get_global_energy_mix_data()[ - self._country_iso_code - ]["country_name"] - except KeyError as e: - logger.error( - "Does not support country" - + f" with ISO code {self._country_iso_code} " - f"Exception occurred {e}" - ) - if self._country_2letter_iso_code: assert isinstance(self._country_2letter_iso_code, str) self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper() super().__init__(*args, **kwargs) + def _resolve_offline_country_name(self) -> None: + if self._country_name is not None or not self._country_iso_code: + return + try: + self._country_name = DataSource().get_global_energy_mix_data()[ + self._country_iso_code + ]["country_name"] + except KeyError as e: + logger.error( + "Does not support country" + f" with ISO code {self._country_iso_code} " + f"Exception occurred {e}" + ) + + def _validate_offline_cloud_provider(self) -> None: + if not self._cloud_provider: + return + df = DataSource().get_cloud_emissions_data() + if ( + len( + df.loc[ + (df["provider"] == self._cloud_provider) + & (df["region"] == self._cloud_region) + ] + ) + == 0 + ): + logger.error( + "Cloud Provider/Region " + f"{self._cloud_provider} {self._cloud_region} " + "not found in cloud emissions data." + ) + def _get_geo_metadata(self) -> GeoMetadata: + from codecarbon.external.geography import GeoMetadata + + self._resolve_offline_country_name() return GeoMetadata( country_iso_code=self._country_iso_code, country_name=self._country_name, @@ -1359,6 +1406,9 @@ def _get_geo_metadata(self) -> GeoMetadata: ) def _get_cloud_metadata(self) -> CloudMetadata: + from codecarbon.external.geography import CloudMetadata + + self._validate_offline_cloud_provider() if self._cloud is None: self._cloud = CloudMetadata( provider=self._cloud_provider, region=self._cloud_region @@ -1373,9 +1423,13 @@ class EmissionsTracker(BaseEmissionsTracker): """ def _get_geo_metadata(self) -> GeoMetadata: + from codecarbon.external.geography import GeoMetadata + return GeoMetadata.from_geo_js(self._data_source.geo_js_url) def _get_cloud_metadata(self) -> CloudMetadata: + from codecarbon.external.geography import CloudMetadata + if self._cloud is None: self._cloud = CloudMetadata.from_utils() return self._cloud diff --git a/codecarbon/external/hardware.py b/codecarbon/external/hardware.py index 8ac4de8f8..5074f69b9 100644 --- a/codecarbon/external/hardware.py +++ b/codecarbon/external/hardware.py @@ -28,6 +28,14 @@ MODE_CPU_LOAD = "cpu_load" +# psutil.cpu_percent blocks on first sample; prime once per process for cpu_load mode. +_cpu_load_percent_primed = False + + +def clear_cpu_load_prime_cache() -> None: + global _cpu_load_percent_primed + _cpu_load_percent_primed = False + @dataclass class BaseHardware(ABC): @@ -210,6 +218,8 @@ def __init__( # For process tracking: store last measurement time and CPU times self._last_measurement_time: Optional[float] = None self._last_cpu_times: Dict[int, float] = {} # pid -> total cpu time + # First cpu_percent sample blocks briefly; later calls use interval=None. + self._cpu_percent_interval: Optional[float] = 0.05 if self._mode == "intel_power_gadget": self._intel_interface = IntelPowerGadget(self._output_dir) @@ -263,8 +273,10 @@ def _get_power_from_cpu_load(self): if self._tracking_mode == "machine": tdp = self._tdp cpu_load = psutil.cpu_percent( - interval=0.5, percpu=False - ) # Convert to 0-1 range + interval=self._cpu_percent_interval, percpu=False + ) + if self._cpu_percent_interval is not None: + self._cpu_percent_interval = None logger.debug(f"CPU load : {self._tdp=} W and {cpu_load:.1f} %") # Cubic relationship with minimum 10% of TDP load_factor = 0.1 + 0.9 * ((cpu_load / 100.0) ** 3) @@ -395,15 +407,18 @@ def measure_power_and_energy(self, last_duration: float) -> Tuple[Power, Energy] return super().measure_power_and_energy(last_duration=last_duration) def start(self): + global _cpu_load_percent_primed if self._mode in ["intel_power_gadget", "intel_rapl", "apple_powermetrics"]: self._intel_interface.start() # Reset process tracking state for fresh measurements self._last_measurement_time = None self._last_cpu_times = {} if self._mode == MODE_CPU_LOAD: - # The first time this is called it will return a meaningless 0.0 value which you are supposed to ignore. - _ = self._get_power_from_cpu_load() - _ = self._get_power_from_cpu_load() + if not _cpu_load_percent_primed: + _ = self._get_power_from_cpu_load() + _cpu_load_percent_primed = True + else: + self._cpu_percent_interval = None def monitor_power(self): cpu_power = self._get_power_from_cpus() @@ -435,6 +450,7 @@ def from_utils( mode=mode, model=model, tdp=tdp, + tracking_mode=tracking_mode, rapl_include_dram=rapl_include_dram, rapl_prefer_psys=rapl_prefer_psys, ) diff --git a/codecarbon/external/task.py b/codecarbon/external/task.py index e3d7ecbae..b8945960e 100644 --- a/codecarbon/external/task.py +++ b/codecarbon/external/task.py @@ -1,7 +1,7 @@ import time from uuid import uuid4 -from codecarbon.output import EmissionsData, TaskEmissionsData +from codecarbon.output_methods.emissions_data import EmissionsData, TaskEmissionsData class Task: diff --git a/codecarbon/input.py b/codecarbon/input.py index 93a96c988..4f705fcb5 100644 --- a/codecarbon/input.py +++ b/codecarbon/input.py @@ -1,11 +1,12 @@ """ App configuration and static reference data loading. -Data files are static reference data that never change during runtime. -They are loaded once at module import to avoid repeated file I/O on the hot path -(start_task/stop_task calls for instance). +Static CSV/JSON reference data is loaded lazily on first DataSource access +to keep `import codecarbon` fast for measurement startup. """ +from __future__ import annotations + import atexit import json from contextlib import ExitStack @@ -13,8 +14,6 @@ from importlib.resources import files as importlib_resources_files from typing import Any, Dict -import pandas as pd - _CACHE: Dict[str, Any] = {} _MODULE_NAME = "codecarbon" @@ -35,6 +34,8 @@ def _load_static_data() -> None: Called once when codecarbon is imported. All data loaded here is immutable and shared across all tracker instances. """ + import pandas as pd + # Global energy mix - used for emissions calculations path = _get_resource_path("data/private_infra/global_energy_mix.json") with open(path) as f: @@ -59,8 +60,16 @@ def _load_static_data() -> None: _CACHE["nordic_country_energy_mix"] = json.load(f) -# Load static data at module import -_load_static_data() +_STATIC_DATA_LOADED = False + + +def _ensure_static_data_loaded() -> None: + """Load immutable reference data on first use instead of at import.""" + global _STATIC_DATA_LOADED + if _STATIC_DATA_LOADED: + return + _load_static_data() + _STATIC_DATA_LOADED = True class DataSource: @@ -130,15 +139,17 @@ def cpu_power_path(self): def get_global_energy_mix_data(self) -> Dict: """ Returns Global Energy Mix Data. - Data is pre-loaded at module import for performance. + Data is loaded on first access and cached for all tracker instances. """ + _ensure_static_data_loaded() return _CACHE["global_energy_mix"] - def get_cloud_emissions_data(self) -> pd.DataFrame: + def get_cloud_emissions_data(self): """ Returns Cloud Regions Impact Data. - Data is pre-loaded at module import for performance. + Data is loaded on first access and cached for all tracker instances. """ + _ensure_static_data_loaded() return _CACHE["cloud_emissions"] def get_country_emissions_data(self, country_iso_code: str) -> Dict: @@ -176,22 +187,25 @@ def get_country_energy_mix_data(self, country_iso_code: str) -> Dict: def get_carbon_intensity_per_source_data(self) -> Dict: """ Returns Carbon intensity per source. In gCO2.eq/kWh. - Data is pre-loaded at module import for performance. + Data is loaded on first access and cached for all tracker instances. """ + _ensure_static_data_loaded() return _CACHE["carbon_intensity_per_source"] - def get_cpu_power_data(self) -> pd.DataFrame: + def get_cpu_power_data(self): """ Returns CPU power Data. - Data is pre-loaded at module import for performance. + Data is loaded on first access and cached for all tracker instances. """ + _ensure_static_data_loaded() return _CACHE["cpu_power"] def get_nordic_country_energy_mix_data(self) -> Dict: """ Returns Nordic Country Energy Mix Data. - Data is cached on first access per country. + Data is loaded on first access and cached for all tracker instances. """ + _ensure_static_data_loaded() return _CACHE["nordic_country_energy_mix"] diff --git a/codecarbon/output_methods/http.py b/codecarbon/output_methods/http.py index 936d6a926..e0ff710b1 100644 --- a/codecarbon/output_methods/http.py +++ b/codecarbon/output_methods/http.py @@ -49,23 +49,28 @@ def __init__( ): self.endpoint_url: str = endpoint_url self.api = ApiClient( - experiment_id=experiment_id, endpoint_url=endpoint_url, + experiment_id=experiment_id, api_key=api_key, conf=conf, + create_run_automatically=False, ) self.run_id = self.api.run_id - def live_out(self, _, delta: EmissionsData): - # Called at regular intervals + def _ensure_api_run(self) -> None: + if self.api.run_id is None and self.api.experiment_id is not None: + self.api._create_run(self.api.experiment_id) + self.run_id = self.api.run_id + + def _emit(self, delta: EmissionsData) -> None: try: + self._ensure_api_run() self.api.add_emission(dataclasses.asdict(delta)) except Exception as e: logger.error(e, exc_info=True) + def live_out(self, _, delta: EmissionsData): + self._emit(delta) + def out(self, _, delta: EmissionsData): - # Called on exit - try: - self.api.add_emission(dataclasses.asdict(delta)) - except Exception as e: - logger.error(e, exc_info=True) + self._emit(delta) diff --git a/docs/explanation/faq.md b/docs/explanation/faq.md index e1fc19fe6..624ebd90e 100644 --- a/docs/explanation/faq.md +++ b/docs/explanation/faq.md @@ -45,6 +45,10 @@ If you find any functionality missing in the CodeCarbon repo, please [open an is By default, CodeCarbon saves emissions data locally. You can configure HTTP output to send data to your own endpoints. We do send data to our API when the user allows it and logs in. No data is sent to third parties without explicit configuration. +## Why is my second tracker faster than the first? + +In a single Python process, the first tracker pays a one-time cost to detect hardware (CPU model, GPU devices, RAM, power backends, and related setup). Later trackers in the same process reuse that cached setup, so `start()` and `stop()` are much faster on warm runs. This is expected: each new process still performs a full cold setup once. + ## What hardware does CodeCarbon support? CodeCarbon supports various CPU architectures, GPUs, and cloud providers. For details on measurement priority and supported hardware, see the [Methodology](methodology.md#cpu-metrics-priority) page. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1e93a588f..71ba05a85 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -44,7 +44,7 @@ Displays real-time emissions data for all processes on your machine. Press `Ctrl | `--no-api` | flag | false | Do not send data to the API (local-only measurement) | | `--offline` | flag | false | Run without internet access | | `--country-iso-code` | string | - | ISO 3166-1 alpha-3 country code (required in offline mode) | -| `--log-level` | choice | INFO | Log level: DEBUG, INFO, WARNING, ERROR | +| `--log-level` | choice | ERROR | Log level: DEBUG, INFO, WARNING, ERROR | **Examples:** ```bash diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 0935cc069..c4f990b4d 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -11,7 +11,7 @@ # MOCK API CLIENT -@patch("codecarbon.cli.main.ApiClient") +@patch("codecarbon.core.api_client.ApiClient") class TestApp(unittest.TestCase): def setUp(self): self.runner = CliRunner() @@ -57,7 +57,7 @@ def test_app(self, MockApiClient): @patch("codecarbon.cli.main.Path.exists") @patch("codecarbon.cli.main.Confirm.ask") @patch("codecarbon.cli.main.questionary_prompt") - @patch("codecarbon.cli.main.get_access_token") + @patch("codecarbon.cli.auth.get_access_token") @patch("typer.prompt") def test_config_no_local_new_all( self, @@ -147,7 +147,7 @@ def side_effect_wrapper(*args, **kwargs): except OSError: pass - @patch("codecarbon.cli.main.get_access_token") + @patch("codecarbon.cli.auth.get_access_token") @patch("codecarbon.cli.main.Path.exists") @patch("codecarbon.cli.main.get_config") @patch("codecarbon.cli.main.questionary_prompt") @@ -186,5 +186,16 @@ def custom_questionary_side_effect(*args, **kwargs): return MagicMock(return_value=default_value) +class TestQuestionaryPrompt(unittest.TestCase): + @patch("questionary.select") + def test_questionary_prompt_returns_selected_value(self, mock_select): + from codecarbon.cli.main import questionary_prompt + + mock_select.return_value.ask.return_value = "selected" + result = questionary_prompt("Pick one", ["a", "b"], "a") + self.assertEqual(result, "selected") + mock_select.assert_called_once_with("Pick one", ["a", "b"], "a") + + if __name__ == "__main__": unittest.main() diff --git a/tests/cli/test_cli_main.py b/tests/cli/test_cli_main.py index 2319dadad..84f42493d 100644 --- a/tests/cli/test_cli_main.py +++ b/tests/cli/test_cli_main.py @@ -3,6 +3,7 @@ from types import SimpleNamespace import pytest +import typer from typer.testing import CliRunner from codecarbon.cli import main as cli_main @@ -34,8 +35,8 @@ def test_version_flag(): def test_api_get_calls_api_and_prints(monkeypatch): runner = CliRunner() - monkeypatch.setattr(cli_main, "ApiClient", FakeApiClient) - monkeypatch.setattr(cli_main, "get_access_token", fake_get_access_token) + monkeypatch.setattr("codecarbon.core.api_client.ApiClient", FakeApiClient) + monkeypatch.setattr("codecarbon.cli.auth.get_access_token", fake_get_access_token) result = runner.invoke(cli_main.codecarbon, ["test-api"]) assert result.exit_code == 0 @@ -51,11 +52,11 @@ def __init__(self, endpoint_url=None): super().__init__(endpoint_url=endpoint_url) runner = CliRunner() - monkeypatch.setattr(cli_main, "ApiClient", CustomApiClient) + monkeypatch.setattr("codecarbon.core.api_client.ApiClient", CustomApiClient) monkeypatch.setattr( cli_main, "get_api_endpoint", lambda: "https://custom.codecarbon.io" ) - monkeypatch.setattr(cli_main, "get_access_token", fake_get_access_token) + monkeypatch.setattr("codecarbon.cli.auth.get_access_token", fake_get_access_token) result = runner.invoke(cli_main.codecarbon, ["test-api"]) assert result.exit_code == 0 @@ -85,7 +86,7 @@ def get_detected_hardware(self): "gpu_ids": None, } - monkeypatch.setattr(cli_main, "EmissionsTracker", FakeTracker) + monkeypatch.setattr("codecarbon.emissions_tracker.EmissionsTracker", FakeTracker) runner = CliRunner() result = runner.invoke(cli_main.codecarbon, ["detect"]) assert result.exit_code == 0 @@ -115,7 +116,7 @@ def set_access_token(self, token): def fake_get_access_token(): raise ValueError("Not able to retrieve the access token, please run login.") - monkeypatch.setattr(cli_main, "ApiClient", FakeApiClient) + monkeypatch.setattr("codecarbon.core.api_client.ApiClient", FakeApiClient) monkeypatch.setattr( cli_main, "get_config", @@ -129,7 +130,7 @@ def fake_get_access_token(): monkeypatch.setattr( cli_main, "get_api_endpoint", lambda path: "https://api.codecarbon.io" ) - monkeypatch.setattr(cli_main, "get_access_token", fake_get_access_token) + monkeypatch.setattr("codecarbon.cli.auth.get_access_token", fake_get_access_token) cli_main.show_config(tmp_path / ".codecarbon.config") captured = capsys.readouterr() @@ -165,16 +166,15 @@ def set_access_token(self, token): def check_auth(self): calls["check_auth"] += 1 - monkeypatch.setattr(cli_main, "ApiClient", FakeApiClient) + monkeypatch.setattr("codecarbon.core.api_client.ApiClient", FakeApiClient) monkeypatch.setattr( - cli_main, - "authorize", + "codecarbon.cli.auth.authorize", lambda: calls.__setitem__("authorize", calls["authorize"] + 1), ) monkeypatch.setattr( cli_main, "get_api_endpoint", lambda: "https://custom-login.codecarbon.io" ) - monkeypatch.setattr(cli_main, "get_access_token", lambda: "login-token") + monkeypatch.setattr("codecarbon.cli.auth.get_access_token", lambda: "login-token") runner = CliRunner() result = runner.invoke(cli_main.codecarbon, ["login"]) @@ -198,8 +198,8 @@ def fake_post(url, json, headers): captured["headers"] = headers return FakeResponse() - monkeypatch.setattr(cli_main, "get_access_token", lambda: "access-token") - monkeypatch.setattr(cli_main.requests, "post", fake_post) + monkeypatch.setattr("codecarbon.cli.auth.get_access_token", lambda: "access-token") + monkeypatch.setattr("requests.post", fake_post) token = cli_main.get_api_key("proj-123") assert token == "project-api-token" @@ -235,8 +235,8 @@ def get_project(self, project_id): def get_experiment(self, experiment_id): return {"id": experiment_id} - monkeypatch.setattr(cli_main, "ApiClient", FakeApiClient) - monkeypatch.setattr(cli_main, "get_access_token", lambda: "fake-token") + monkeypatch.setattr("codecarbon.core.api_client.ApiClient", FakeApiClient) + monkeypatch.setattr("codecarbon.cli.auth.get_access_token", lambda: "fake-token") monkeypatch.setattr( cli_main, "get_api_endpoint", lambda path: "https://api.codecarbon.io" ) @@ -289,7 +289,9 @@ def start(self): def stop(self): return None - monkeypatch.setattr(cli_main, "OfflineEmissionsTracker", FakeOfflineTracker) + monkeypatch.setattr( + "codecarbon.emissions_tracker.OfflineEmissionsTracker", FakeOfflineTracker + ) monkeypatch.setattr(cli_main.signal, "signal", lambda *args, **kwargs: None) runner = CliRunner() @@ -311,7 +313,7 @@ def fake_run_and_monitor(ctx, offline=False, **kwargs): captured["kwargs"] = kwargs return "ok" - monkeypatch.setattr(cli_main, "run_and_monitor", fake_run_and_monitor) + monkeypatch.setattr("codecarbon.cli.monitor.run_and_monitor", fake_run_and_monitor) ctx = SimpleNamespace(args=["python", "-c", "print(1)"]) result = cli_main.monitor( @@ -332,7 +334,7 @@ def fake_run_and_monitor(ctx, offline=False, **kwargs): captured["kwargs"] = kwargs return "ok" - monkeypatch.setattr(cli_main, "run_and_monitor", fake_run_and_monitor) + monkeypatch.setattr("codecarbon.cli.monitor.run_and_monitor", fake_run_and_monitor) monkeypatch.setattr(cli_main, "get_existing_exp_id", lambda: "exp-1") ctx = SimpleNamespace(args=["python", "train.py"]) @@ -350,7 +352,7 @@ def fake_run_and_monitor(ctx, **kwargs): captured["kwargs"] = kwargs return "ok" - monkeypatch.setattr(cli_main, "run_and_monitor", fake_run_and_monitor) + monkeypatch.setattr("codecarbon.cli.monitor.run_and_monitor", fake_run_and_monitor) monkeypatch.setattr(cli_main, "get_existing_exp_id", lambda: "exp-1") ctx = SimpleNamespace(args=["python", "train.py"]) @@ -368,7 +370,7 @@ def fake_run_and_monitor(ctx, offline=False, **kwargs): captured["kwargs"] = kwargs return "ok" - monkeypatch.setattr(cli_main, "run_and_monitor", fake_run_and_monitor) + monkeypatch.setattr("codecarbon.cli.monitor.run_and_monitor", fake_run_and_monitor) monkeypatch.setattr(cli_main, "get_existing_exp_id", lambda: None) ctx = SimpleNamespace(args=["python", "train.py"]) @@ -376,3 +378,31 @@ def fake_run_and_monitor(ctx, offline=False, **kwargs): assert result == "ok" assert captured["offline"] is False assert captured["kwargs"]["save_to_api"] is False + + +def test_monitor_passes_log_level_to_run_and_monitor(monkeypatch): + captured = {} + + def fake_run_and_monitor(ctx, offline=False, **kwargs): + captured["kwargs"] = kwargs + + monkeypatch.setattr("codecarbon.cli.monitor.run_and_monitor", fake_run_and_monitor) + + ctx = SimpleNamespace(args=["echo", "hello"]) + cli_main.monitor( + ctx=ctx, + offline=True, + country_iso_code="FRA", + log_level="debug", + ) + + assert captured["kwargs"]["log_level"] == "debug" + + +def test_monitor_online_requires_experiment_id_for_wrapped_command(monkeypatch): + monkeypatch.setattr(cli_main, "get_existing_exp_id", lambda: None) + + ctx = SimpleNamespace(args=["echo", "hi"]) + with pytest.raises(typer.Exit) as exc_info: + cli_main.monitor(ctx=ctx, offline=False, api=True) + assert exc_info.value.exit_code == 1 diff --git a/tests/cli/test_monitor.py b/tests/cli/test_monitor.py index d4dd718a2..0a9bda365 100644 --- a/tests/cli/test_monitor.py +++ b/tests/cli/test_monitor.py @@ -20,8 +20,15 @@ def stop(self): return 0.123 +def _patch_trackers(monkeypatch, online_cls=FakeTracker, offline_cls=FakeTracker): + monkeypatch.setattr("codecarbon.emissions_tracker.EmissionsTracker", online_cls) + monkeypatch.setattr( + "codecarbon.emissions_tracker.OfflineEmissionsTracker", offline_cls + ) + + def test_run_and_monitor_requires_command(monkeypatch): - monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + _patch_trackers(monkeypatch) monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) with pytest.raises(typer.Exit) as exc_info: @@ -30,12 +37,35 @@ def test_run_and_monitor_requires_command(monkeypatch): assert exc_info.value.exit_code == 1 +def test_run_and_monitor_strips_nested_monitor_prefix(monkeypatch): + captured = {} + + class FakePopen: + def __init__(self, command, text=True): + captured["command"] = command + + def wait(self): + return 0 + + _patch_trackers(monkeypatch) + monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) + monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) + + with pytest.raises(typer.Exit) as exc_info: + monitor_module.run_and_monitor( + SimpleNamespace(args=["monitor", "--", "echo", "hi"]) + ) + + assert exc_info.value.exit_code == 0 + assert captured["command"] == ["echo", "hi"] + + def test_run_and_monitor_handles_missing_command(monkeypatch): class FakePopen: def __init__(self, command, text=True): raise FileNotFoundError - monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + _patch_trackers(monkeypatch) monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) @@ -50,7 +80,7 @@ class FakePopen: def __init__(self, command, text=True): raise RuntimeError("boom") - monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + _patch_trackers(monkeypatch) monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) @@ -75,8 +105,7 @@ def __init__(self, command, text=True): def wait(self): return 0 - monkeypatch.setattr(monitor_module, "OfflineEmissionsTracker", FakeOfflineTracker) - monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + _patch_trackers(monkeypatch, offline_cls=FakeOfflineTracker) monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) @@ -106,8 +135,7 @@ def __init__(self, command, text=True): def wait(self): return 0 - monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeOnlineTracker) - monkeypatch.setattr(monitor_module, "OfflineEmissionsTracker", FakeTracker) + _patch_trackers(monkeypatch, online_cls=FakeOnlineTracker) monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) @@ -140,7 +168,7 @@ def terminate(self): def kill(self): process_info["killed"] += 1 - monkeypatch.setattr(monitor_module, "EmissionsTracker", FakeTracker) + _patch_trackers(monkeypatch) monkeypatch.setattr(monitor_module.subprocess, "Popen", FakePopen) monkeypatch.setattr(monitor_module, "print", lambda *args, **kwargs: None) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..ede903900 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +"""Shared pytest fixtures for the CodeCarbon test suite.""" + +import pytest + +from codecarbon.core.hardware_cache import clear_cache as clear_hardware_cache + + +@pytest.fixture(autouse=True) +def _reset_process_hardware_cache(): + """Isolate hardware/TDP/GPU probe caches between tests.""" + from codecarbon.core.util import detect_cpu_model + + clear_hardware_cache() + detect_cpu_model.cache_clear() + yield + clear_hardware_cache() + detect_cpu_model.cache_clear() diff --git a/tests/output_methods/test_http.py b/tests/output_methods/test_http.py index 790055c0a..56d909b46 100644 --- a/tests/output_methods/test_http.py +++ b/tests/output_methods/test_http.py @@ -147,6 +147,45 @@ def test_codecarbon_api_live_out(self): api_output.live_out(None, self.emissions_data) self.mock_add_emission.assert_called_once() + def test_codecarbon_api_live_out_creates_run_when_missing(self): + conf = { + "os": "linux", + "python_version": "3.12", + "codecarbon_version": "2.0", + "cpu_count": 4, + "cpu_model": "CPU", + "gpu_count": 0, + "gpu_model": "", + "longitude": 0.0, + "latitude": 0.0, + "region": "EU", + "provider": "AWS", + "ram_total_size": 16.0, + "tracking_mode": "machine", + } + + with patch( + "codecarbon.output_methods.http.ApiClient._create_run" + ) as mock_create_run: + api_output = CodeCarbonAPIOutput( + endpoint_url="http://test.com", + experiment_id="exp-1", + api_key=self.api_key, + conf=conf, + ) + api_output.api.run_id = None + + def create_run(experiment_id): + api_output.api.run_id = "run-created" + return "run-created" + + mock_create_run.side_effect = create_run + api_output.live_out(None, self.emissions_data) + + mock_create_run.assert_called_once_with("exp-1") + self.assertEqual(api_output.api.run_id, "run-created") + self.assertEqual(api_output.run_id, "run-created") + @patch("codecarbon.output_methods.http.logger.error") def test_codecarbon_live_out_api_call_failure(self, mock_logger): self.mock_add_emission.side_effect = Exception("Test exception") diff --git a/tests/test_config.py b/tests/test_config.py index ef66f66f8..5efb0821d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -27,9 +27,13 @@ def setUp(self): "CODECARBON_API_KEY", "CODECARBON_EXPERIMENT_ID", "CODECARBON_API_ENDPOINT", + "CODECARBON_TELEMETRY", + "CODECARBON_TELEMETRY_PROJECT_TOKEN", "codecarbon_api_key", "codecarbon_experiment_id", "codecarbon_api_endpoint", + "codecarbon_telemetry", + "codecarbon_telemetry_project_token", ]: os.environ.pop(key, None) os.environ.setdefault("CODECARBON_ALLOW_MULTIPLE_RUNS", "True") diff --git a/tests/test_cpu.py b/tests/test_cpu.py index b9acb5b59..e1010a5c9 100644 --- a/tests/test_cpu.py +++ b/tests/test_cpu.py @@ -35,7 +35,24 @@ class TestCPU(unittest.TestCase): def test_is_powergadget_available_returns_false_on_exception( self, mock_powergadget ): + from codecarbon.core.cpu import clear_powergadget_cache + + clear_powergadget_cache() self.assertFalse(is_powergadget_available()) + clear_powergadget_cache() + + def test_is_powergadget_available_returns_cached_value(self): + from codecarbon.core.cpu import clear_powergadget_cache + + clear_powergadget_cache() + with mock.patch("codecarbon.core.cpu.IntelPowerGadget"): + self.assertTrue(is_powergadget_available()) + with mock.patch( + "codecarbon.core.cpu.IntelPowerGadget", + side_effect=Exception("should not instantiate"), + ): + self.assertTrue(is_powergadget_available()) + clear_powergadget_cache() @mock.patch("psutil.cpu_times") def test_is_psutil_available_with_nice(self, mock_cpu_times): @@ -294,7 +311,7 @@ def test_log_values_warns_on_nonzero_returncode_windows(self): mock_warning.assert_called_once() @mock.patch("codecarbon.core.cpu.IntelPowerGadget._log_values") - @mock.patch("codecarbon.core.cpu.pd.read_csv", side_effect=Exception("bad csv")) + @mock.patch("pandas.read_csv", side_effect=Exception("bad csv")) @mock.patch("codecarbon.core.cpu.IntelPowerGadget._setup_cli") def test_get_cpu_details_returns_empty_dict_on_read_error( self, mock_setup, mock_read_csv, mock_log_values @@ -374,7 +391,7 @@ def test_get_cpu_power_from_registry(self): def test_get_cpu_power_from_registry_returns_none_without_match(self): tdp = TDP.__new__(TDP) with ( - mock.patch("codecarbon.core.cpu.DataSource") as mock_data_source, + mock.patch("codecarbon.input.DataSource") as mock_data_source, mock.patch.object(tdp, "_get_matching_cpu", return_value=None), ): mock_data_source.return_value.get_cpu_power_data.return_value = ( @@ -597,11 +614,20 @@ def __init__(self): with ( mock.patch( - "codecarbon.core.resource_tracker.cpu.TDP", + "codecarbon.core.resource_tracker.get_cached_tdp", side_effect=AssertionError( "TDP should not be instantiated when RAPL is active" ), ) as mocked_tdp, + mock.patch( + "codecarbon.core.resource_tracker.is_linux_os", return_value=True + ), + mock.patch( + "codecarbon.core.resource_tracker.is_mac_os", return_value=False + ), + mock.patch( + "codecarbon.core.resource_tracker.is_windows_os", return_value=False + ), mock.patch( "codecarbon.core.resource_tracker.cpu.is_powergadget_available", return_value=False, @@ -625,6 +651,7 @@ def __init__(self): mocked_from_utils.assert_called_once_with( output_dir=tracker._output_dir, mode="intel_rapl", + tracking_mode=tracker._tracking_mode, rapl_include_dram=tracker._rapl_include_dram, rapl_prefer_psys=tracker._rapl_prefer_psys, ) @@ -650,7 +677,8 @@ def __init__(self): with ( mock.patch( - "codecarbon.core.resource_tracker.cpu.TDP", return_value=fake_tdp + "codecarbon.core.resource_tracker.get_cached_tdp", + return_value=fake_tdp, ) as mocked_tdp, mock.patch( "codecarbon.core.resource_tracker.ResourceTracker._setup_cpu_load_mode", @@ -674,7 +702,7 @@ def __init__(self): ): resource_tracker.set_CPU_tracking() - mocked_tdp.assert_called_once_with() + mocked_tdp.assert_called_once() mocked_setup_cpu_load.assert_called_once_with(fake_tdp, 100) mocked_fallback.assert_not_called() @@ -697,11 +725,21 @@ def __init__(self): with ( mock.patch( - "codecarbon.core.resource_tracker.cpu.TDP", return_value=fake_tdp + "codecarbon.core.resource_tracker.get_cached_tdp", + return_value=fake_tdp, ) as mocked_tdp, mock.patch( "codecarbon.core.resource_tracker.ResourceTracker._setup_fallback_tracking" ) as mocked_fallback, + mock.patch( + "codecarbon.core.resource_tracker.is_mac_os", return_value=False + ), + mock.patch( + "codecarbon.core.resource_tracker.is_linux_os", return_value=False + ), + mock.patch( + "codecarbon.core.resource_tracker.is_windows_os", return_value=False + ), mock.patch( "codecarbon.core.resource_tracker.cpu.is_powergadget_available", return_value=False, @@ -717,7 +755,7 @@ def __init__(self): ): resource_tracker.set_CPU_tracking() - mocked_tdp.assert_called_once_with() + mocked_tdp.assert_called_once() mocked_fallback.assert_called_once_with(fake_tdp, 80) diff --git a/tests/test_cpu_load.py b/tests/test_cpu_load.py index f5cdf7e46..b4e44b66d 100644 --- a/tests/test_cpu_load.py +++ b/tests/test_cpu_load.py @@ -56,6 +56,7 @@ def test_cpu_load_detection( mocked_is_rapl_available, ): tracker = OfflineEmissionsTracker(country_iso_code="FRA") + tracker._ensure_hardware_ready() for hardware in tracker._hardware: if ( isinstance(hardware, CPU) and hardware._mode == MODE_CPU_LOAD diff --git a/tests/test_custom_handler.py b/tests/test_custom_handler.py index 570d2df73..8adcf7c37 100644 --- a/tests/test_custom_handler.py +++ b/tests/test_custom_handler.py @@ -32,7 +32,8 @@ def test_carbon_tracker_custom_handler(self): tracker = EmissionsTracker( project_name=self.project_name, output_handlers=[handler_0, handler_1], - api_call_interval=1, + api_call_interval=2, + measure_power_secs=999, ) tracker.start() heavy_computation(run_time_secs=1) @@ -52,7 +53,8 @@ def test_decorator_flush(self): project_name=self.project_name, save_to_logger=True, output_handlers=[handler_0, handler_1], - api_call_interval=1, + api_call_interval=2, + measure_power_secs=999, ) def dummy_train_model(): heavy_computation(run_time_secs=1) diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index 37f1000de..0715b665c 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -605,6 +605,30 @@ def test_carbon_tracker_online_context_manager_TWO_GPU_PRIVATE_INFRA_CANADA( self.assertIsInstance(tracker.final_emissions, float) self.assertAlmostEqual(tracker.final_emissions, 6.262572537957655e-05, places=2) + def test_start_task_returns_when_engine_initialization_fails( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + tracker = EmissionsTracker(save_to_file=False) + with ( + mock.patch.object( + tracker, + "_ensure_emissions_engine", + side_effect=Exception("init failed"), + ), + self.assertLogs("codecarbon", level="ERROR") as logs, + ): + tracker.start_task("failed-task") + + self.assertTrue( + any("Tracker not initialized" in message for message in logs.output) + ) + @mock.patch("codecarbon.external.ram.RAM.measure_power_and_energy") @mock.patch("codecarbon.external.hardware.CPU.measure_power_and_energy") @mock.patch( @@ -851,7 +875,7 @@ def test_get_detected_hardware( @mock.patch("codecarbon.emissions_tracker.EmissionsTracker._get_geo_metadata") @mock.patch("codecarbon.emissions_tracker.EmissionsTracker._get_cloud_metadata") @mock.patch("codecarbon.core.electricitymaps_api.requests.get") - @mock.patch("codecarbon.emissions_tracker.ResourceTracker") + @mock.patch("codecarbon.core.resource_tracker.ResourceTracker") @mock.patch( "codecarbon.emissions_tracker.BaseEmissionsTracker.get_detected_hardware" ) @@ -919,10 +943,9 @@ def test_cumulative_emissions_with_varying_intensity( ) tracker._hardware = [mock_cpu] - # Start tracking + # Start tracking (includes an immediate first measurement) tracker.start() - tracker._measure_power_and_energy() # total_energy = 1.0, intensity = 100 => emissions = 0.1 kg data1 = tracker._prepare_emissions_data() self.assertAlmostEqual(data1.emissions, 0.1) diff --git a/tests/test_emissions_tracker_constant.py b/tests/test_emissions_tracker_constant.py index 65b17c666..724cbe9ab 100644 --- a/tests/test_emissions_tracker_constant.py +++ b/tests/test_emissions_tracker_constant.py @@ -5,7 +5,6 @@ from unittest import mock import pandas as pd -import psutil from codecarbon.core import cpu from codecarbon.emissions_tracker import ( @@ -89,14 +88,15 @@ def test_carbon_tracker_offline_constant_force_cpu_power( assertdf = pd.read_csv(self.emissions_file_path) self.assertEqual(USER_INPUT_CPU_POWER / 2, assertdf["cpu_power"][0]) + @mock.patch("codecarbon.external.hardware.psutil.cpu_percent", return_value=50.0) @mock.patch.object(cpu.TDP, "_get_cpu_power_from_registry") @mock.patch.object(cpu, "is_psutil_available") - def test_carbon_tracker_offline_load_force_cpu_power(self, mock_tdp, mock_psutil): - # Same as test_carbon_tracker_offline_constant test but this time forcing the default cpu power + def test_carbon_tracker_offline_load_force_cpu_power( + self, mock_psutil_available, mock_tdp, mock_cpu_percent + ): USER_INPUT_CPU_POWER = 1_000 - # Mock the output of tdp mock_tdp.return_value = 500 - mock_psutil.return_value = True + mock_psutil_available.return_value = True tracker = OfflineEmissionsTracker( country_iso_code="USA", output_dir=self.emissions_path, @@ -108,17 +108,11 @@ def test_carbon_tracker_offline_load_force_cpu_power(self, mock_tdp, mock_psutil emissions = tracker.stop() assert isinstance(emissions, float) self.assertNotEqual(emissions, 0.0) - # Get CPU load (measured after test; may differ from load during test) - cpu_load = psutil.cpu_percent(interval=1) / 100.0 - # Assert the content stored. cpu_power should be approximately load * min(TDP, forced CPU power) + cpu_load = 0.5 assertdf = pd.read_csv(self.emissions_file_path) - tolerance = 350 - self.assertLess( - assertdf["cpu_power"][0], USER_INPUT_CPU_POWER * cpu_load + tolerance - ) - self.assertGreater( - assertdf["cpu_power"][0], USER_INPUT_CPU_POWER * cpu_load - tolerance - ) + load_factor = 0.1 + 0.9 * (cpu_load**3) + expected_power = USER_INPUT_CPU_POWER * load_factor + self.assertAlmostEqual(assertdf["cpu_power"][0], expected_power, delta=50) def test_decorator_constant(self): @track_emissions( @@ -151,6 +145,7 @@ def test_carbon_tracker_offline_region_error(self): ) tracker.start() tracker._measure_power_and_energy() + tracker._ensure_emissions_engine() cloud: CloudMetadata = tracker._get_cloud_metadata() try: diff --git a/tests/test_gpu.py b/tests/test_gpu.py index bfbc8e603..7326b3866 100644 --- a/tests/test_gpu.py +++ b/tests/test_gpu.py @@ -159,6 +159,13 @@ def check_output(cmd, *args, **kwargs): class TestGpuMethods: + def setup_method(self): + from codecarbon.core.gpu_amd import clear_rocm_system_cache + from codecarbon.core.gpu_nvidia import clear_nvidia_system_cache + + clear_rocm_system_cache() + clear_nvidia_system_cache() + @mock.patch("codecarbon.core.gpu_amd.subprocess.check_output") def test_is_rocm_system(self, mock_subprocess): from codecarbon.core.gpu import is_rocm_system diff --git a/tests/test_hardware_cache.py b/tests/test_hardware_cache.py new file mode 100644 index 000000000..81d203dbf --- /dev/null +++ b/tests/test_hardware_cache.py @@ -0,0 +1,251 @@ +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from codecarbon.core import hardware_cache +from codecarbon.external.hardware import CPU +from codecarbon.external.ram import RAM + + +def make_tracker(**overrides): + defaults = { + "_tracking_mode": "machine", + "_force_cpu_power": None, + "_force_ram_power": None, + "_conf": {}, + "_gpu_ids": None, + "_rapl_include_dram": False, + "_rapl_prefer_psys": False, + "_output_dir": "out", + "_hardware": [], + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def test_make_key_normalizes_gpu_ids(): + tracker = make_tracker(_gpu_ids=[0, 1]) + key = hardware_cache.make_key(tracker) + assert key.gpu_ids == ("0", "1") + + +def test_make_key_treats_equivalent_gpu_id_types_as_same_key(): + key_int = hardware_cache.make_key(make_tracker(_gpu_ids=[0])) + key_str = hardware_cache.make_key(make_tracker(_gpu_ids=["0"])) + assert key_int == key_str + + +def test_spec_and_rebuild_roundtrip_for_cpu(): + cpu_hw = CPU.from_utils("out", "cpu_load", "Test CPU", 100) + spec = hardware_cache._spec_from_hardware(cpu_hw) + rebuilt = hardware_cache._hardware_from_spec(spec, "out2") + assert type(rebuilt).__name__ == "CPU" + assert rebuilt._model == "Test CPU" + assert rebuilt._mode == "cpu_load" + + +def test_spec_from_hardware_gpu_and_rapl_cpu(): + gpu_hw = type("GPU", (), {"gpu_ids": [0, 1]})() + assert hardware_cache._spec_from_hardware(gpu_hw) == { + "kind": "gpu", + "gpu_ids": ["0", "1"], + } + + gpu_hw_no_ids = type("GPU", (), {"gpu_ids": None})() + assert hardware_cache._spec_from_hardware(gpu_hw_no_ids) == { + "kind": "gpu", + "gpu_ids": None, + } + + gpu_hw_empty_ids = type("GPU", (), {"gpu_ids": []})() + assert hardware_cache._spec_from_hardware(gpu_hw_empty_ids) == { + "kind": "gpu", + "gpu_ids": None, + } + + +def test_capture_serializes_gpu_hardware(): + gpu_hw = type("GPU", (), {"gpu_ids": (0, 1)})() + tracker = make_tracker(_hardware=[gpu_hw]) + resource_tracker = SimpleNamespace( + tracker=tracker, + ram_tracker="ram", + cpu_tracker="cpu", + gpu_tracker="gpu", + ) + + plan = hardware_cache.capture(resource_tracker) + + assert plan.hardware_specs == [{"kind": "gpu", "gpu_ids": ["0", "1"]}] + + +def test_hardware_kind_apple_chip(): + apple_hw = type("AppleSiliconChip", (), {})() + assert hardware_cache._hardware_kind(apple_hw) == "apple_chip" + + +def test_spec_from_hardware_apple_chip(): + apple_hw = type( + "AppleSiliconChip", + (), + {"_model": "Apple M1", "chip_part": "CPU"}, + )() + assert hardware_cache._spec_from_hardware(apple_hw) == { + "kind": "apple_chip", + "model": "Apple M1", + "chip_part": "CPU", + } + + +def test_hardware_from_spec_rebuilds_gpu(): + fake_gpu = SimpleNamespace(gpu_ids=[0]) + with patch( + "codecarbon.external.hardware.GPU.from_utils", + return_value=fake_gpu, + ) as mock_from_utils: + rebuilt = hardware_cache._hardware_from_spec( + {"kind": "gpu", "gpu_ids": ["0"]}, + "out", + ) + mock_from_utils.assert_called_once_with(gpu_ids=["0"]) + assert rebuilt is fake_gpu + + +def test_hardware_from_spec_rejects_unknown_kind(): + with pytest.raises(ValueError, match="Unknown hardware spec kind"): + hardware_cache._hardware_from_spec({"kind": "unknown"}, "out") + + +def test_spec_from_hardware_intel_rapl_cpu(): + cpu_hw = type( + "CPU", + (), + { + "_mode": "intel_rapl", + "_model": "Intel CPU", + "_tdp": 65, + "_tracking_mode": "machine", + "_intel_interface": SimpleNamespace( + rapl_include_dram=True, + rapl_prefer_psys=True, + ), + }, + )() + spec = hardware_cache._spec_from_hardware(cpu_hw) + assert spec["rapl_include_dram"] is True + assert spec["rapl_prefer_psys"] is True + assert spec["rapl_dir"] == "/sys/class/powercap/intel-rapl/subsystem" + + +def test_spec_and_rebuild_roundtrip_for_apple_chip(): + spec = {"kind": "apple_chip", "model": "Apple M1", "chip_part": "CPU"} + fake_chip = SimpleNamespace(_model="Apple M1") + with patch( + "codecarbon.external.hardware.AppleSiliconChip", + return_value=fake_chip, + ) as mock_chip_cls: + rebuilt = hardware_cache._hardware_from_spec(spec, "out") + mock_chip_cls.assert_called_once_with( + output_dir="out", + model="Apple M1", + chip_part="CPU", + ) + assert rebuilt._model == "Apple M1" + + +def test_capture_and_apply_restore_hardware_plan(): + tracker = make_tracker( + _conf={ + "cpu_count": 8, + "cpu_physical_count": 4, + "cpu_model": "Cached CPU", + "gpu_count": 0, + "gpu_model": "", + "gpu_ids": ["0"], + }, + _gpu_ids=[0], + _hardware=[RAM(tracking_mode="machine")], + ) + resource_tracker = SimpleNamespace( + tracker=tracker, + ram_tracker="cached_ram", + cpu_tracker="cached_cpu", + gpu_tracker="cached_gpu", + ) + plan = hardware_cache.capture(resource_tracker) + + tracker2 = make_tracker() + rt2 = SimpleNamespace( + tracker=tracker2, + ram_tracker="old", + cpu_tracker="old", + gpu_tracker="old", + ) + hardware_cache.apply(rt2, plan) + + assert rt2.ram_tracker == "cached_ram" + assert rt2.cpu_tracker == "cached_cpu" + assert tracker2._conf["cpu_model"] == "Cached CPU" + assert tracker2._conf["cpu_count"] == 8 + assert tracker2._conf["cpu_physical_count"] == 4 + assert tracker2._gpu_ids == ["0"] + assert len(tracker2._hardware) == 1 + assert type(tracker2._hardware[0]).__name__ == "RAM" + + +def test_get_or_run_setup_runs_setup_once(): + tracker = make_tracker() + resource_tracker = SimpleNamespace( + tracker=tracker, + ram_tracker="Unspecified", + cpu_tracker="Unspecified", + gpu_tracker="Unspecified", + ) + calls = {"count": 0} + + def setup_fn(): + calls["count"] += 1 + resource_tracker.ram_tracker = "ran" + + hardware_cache.clear_cache() + hardware_cache.get_or_run_setup(resource_tracker, setup_fn) + hardware_cache.get_or_run_setup(resource_tracker, setup_fn) + + assert calls["count"] == 1 + assert resource_tracker.ram_tracker == "ran" + + +def test_hardware_kind_rejects_unknown_type(): + with pytest.raises(TypeError): + hardware_cache._hardware_kind(object()) + + +def test_clear_cache_resets_probe_caches(): + from codecarbon.core.cpu import clear_powergadget_cache, is_powergadget_available + from codecarbon.core.powermetrics import ( + clear_powermetrics_cache, + is_powermetrics_available, + ) + + clear_powergadget_cache() + clear_powermetrics_cache() + with patch("codecarbon.core.cpu.IntelPowerGadget", side_effect=Exception("nope")): + assert is_powergadget_available() is False + with patch( + "codecarbon.core.powermetrics.ApplePowermetrics", side_effect=Exception("nope") + ): + assert is_powermetrics_available() is False + + hardware_cache.clear_cache() + + assert is_powergadget_available.cache_info().currsize == 0 + assert is_powermetrics_available.cache_info().currsize == 0 + + +def test_get_cached_tdp_reuses_instance(): + hardware_cache.clear_cache() + fake_cpu = SimpleNamespace(TDP=lambda: SimpleNamespace(model="cached")) + first = hardware_cache.get_cached_tdp(fake_cpu) + second = hardware_cache.get_cached_tdp(fake_cpu) + assert first is second diff --git a/tests/test_input.py b/tests/test_input.py index 89739d490..875e7e99a 100644 --- a/tests/test_input.py +++ b/tests/test_input.py @@ -12,10 +12,13 @@ class TestDataSourceCaching(unittest.TestCase): """Test that DataSource uses module-level cache for static data.""" def test_cache_populated_at_import(self): - """Verify that _CACHE is populated when module is imported.""" - from codecarbon.input import _CACHE + """Verify that _CACHE is populated on first data access.""" + from codecarbon.input import _CACHE, DataSource + + ds = DataSource() + ds.get_global_energy_mix_data() - # All static data should be pre-loaded + # Static data should be loaded after first access self.assertIn("global_energy_mix", _CACHE) self.assertIn("cloud_emissions", _CACHE) self.assertIn("carbon_intensity_per_source", _CACHE) diff --git a/tests/test_offline_emissions_tracker.py b/tests/test_offline_emissions_tracker.py index 07adf403c..36447409d 100644 --- a/tests/test_offline_emissions_tracker.py +++ b/tests/test_offline_emissions_tracker.py @@ -67,3 +67,14 @@ def test_offline_tracker_task(self): self.assertGreater(task_emission_data.emissions, 0.0) self.assertEqual(task_emission_data.country_name, None) + + def test_resolve_offline_country_name_logs_on_invalid_iso(self): + tracker = OfflineEmissionsTracker( + country_iso_code="INVALID", + save_to_file=False, + ) + with self.assertLogs("codecarbon", level="ERROR") as logs: + tracker._resolve_offline_country_name() + self.assertTrue( + any("Does not support country" in message for message in logs.output) + ) diff --git a/tests/test_powermetrics.py b/tests/test_powermetrics.py index 2fbcba431..b20f5df2c 100644 --- a/tests/test_powermetrics.py +++ b/tests/test_powermetrics.py @@ -15,6 +15,32 @@ def __init__(self, stderr="", returncode=0): def communicate(self): return ("", self._stderr) + def poll(self): + return self.returncode + + def kill(self): + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +class HangingProcess: + def __init__(self): + self.killed = False + + def poll(self): + return None + + def kill(self): + self.killed = True + + def communicate(self): + return ("", "") + def __enter__(self): return self @@ -48,11 +74,68 @@ def test_get_details(self, mock_setup, mock_log_values): assert cpu_details == expected_details def test_is_powermetrics_available_returns_false_on_instantiation_error(self): + from codecarbon.core.powermetrics import clear_powermetrics_cache + + clear_powermetrics_cache() with mock.patch( "codecarbon.core.powermetrics.ApplePowermetrics", side_effect=Exception("boom"), ): assert is_powermetrics_available() is False + clear_powermetrics_cache() + + def test_is_powermetrics_available_returns_cached_value(self): + from codecarbon.core.powermetrics import clear_powermetrics_cache + + clear_powermetrics_cache() + with ( + mock.patch("codecarbon.core.powermetrics.ApplePowermetrics"), + mock.patch( + "codecarbon.core.powermetrics._has_powermetrics_sudo", + return_value=True, + ), + ): + assert is_powermetrics_available() is True + with mock.patch( + "codecarbon.core.powermetrics.ApplePowermetrics", + side_effect=Exception("should not instantiate"), + ): + assert is_powermetrics_available() is True + clear_powermetrics_cache() + + def test_is_powermetrics_available_probes_sudo_when_uncached(self): + from codecarbon.core.powermetrics import clear_powermetrics_cache + + clear_powermetrics_cache() + with ( + mock.patch("codecarbon.core.powermetrics.ApplePowermetrics"), + mock.patch( + "codecarbon.core.powermetrics._has_powermetrics_sudo", + return_value=True, + ) as mock_sudo, + ): + assert is_powermetrics_available() is True + mock_sudo.assert_called_once() + clear_powermetrics_cache() + + def test_has_powermetrics_sudo_kills_process_on_timeout(self): + hanging = HangingProcess() + with ( + mock.patch( + "codecarbon.core.powermetrics.shutil.which", + side_effect=["sudo-path", "powermetrics-path"], + ), + mock.patch( + "codecarbon.core.powermetrics.subprocess.Popen", + return_value=hanging, + ), + mock.patch( + "codecarbon.core.powermetrics.time.time", side_effect=[0, 0, 10] + ), + mock.patch("codecarbon.core.powermetrics.time.sleep"), + ): + assert powermetrics_module._has_powermetrics_sudo() is False + assert hanging.killed is True def test_has_powermetrics_sudo_returns_false_when_sudo_missing(self): with mock.patch("codecarbon.core.powermetrics.shutil.which", return_value=None): diff --git a/tests/test_resource_tracker.py b/tests/test_resource_tracker.py index 632aee464..417bf7bba 100644 --- a/tests/test_resource_tracker.py +++ b/tests/test_resource_tracker.py @@ -204,7 +204,7 @@ def test_set_cpu_tracking_force_mode_uses_cpu_load_and_returns(): fake_tdp = SimpleNamespace(tdp=20, model="CPU") with ( - patch("codecarbon.core.resource_tracker.cpu.TDP", return_value=fake_tdp), + patch("codecarbon.core.resource_tracker.get_cached_tdp", return_value=fake_tdp), patch.object( resource_tracker, "_setup_cpu_load_mode", return_value=True ) as mock_setup, @@ -219,6 +219,9 @@ def test_set_cpu_tracking_prefers_power_gadget(): resource_tracker = ResourceTracker(tracker) with ( + patch("codecarbon.core.resource_tracker.is_mac_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_linux_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_windows_os", return_value=True), patch( "codecarbon.core.resource_tracker.cpu.is_powergadget_available", return_value=True, @@ -237,11 +240,134 @@ def test_set_cpu_tracking_prefers_power_gadget(): mock_power_gadget.assert_called_once_with() +def test_set_cpu_tracking_mac_arm_prefers_cpu_load_over_powermetrics(): + tracker = make_tracker() + resource_tracker = ResourceTracker(tracker) + + with ( + patch("codecarbon.core.resource_tracker.is_mac_os", return_value=True), + patch("codecarbon.core.resource_tracker.is_linux_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_windows_os", return_value=False), + patch( + "codecarbon.core.resource_tracker.detect_cpu_model", + return_value="Apple M1 Max", + ), + patch( + "codecarbon.core.resource_tracker.cpu.is_powergadget_available", + return_value=True, + ), + patch( + "codecarbon.core.resource_tracker.powermetrics.is_powermetrics_available", + return_value=True, + ), + patch.object(resource_tracker, "_setup_power_gadget") as mock_power_gadget, + patch.object(resource_tracker, "_setup_powermetrics") as mock_powermetrics, + patch.object( + resource_tracker, "_setup_cpu_load_fast", return_value=True + ) as mock_cpu_load, + ): + resource_tracker.set_CPU_tracking() + + mock_power_gadget.assert_not_called() + mock_powermetrics.assert_not_called() + mock_cpu_load.assert_called_once_with("Apple M1 Max") + + +def test_setup_cpu_load_fast_returns_false_without_psutil(): + tracker = make_tracker() + resource_tracker = ResourceTracker(tracker) + + with patch( + "codecarbon.core.resource_tracker.cpu.is_psutil_available", + return_value=False, + ): + assert resource_tracker._setup_cpu_load_fast("Intel CPU") is False + + +def test_try_platform_cpu_backend_mac_intel_uses_power_gadget(): + tracker = make_tracker() + resource_tracker = ResourceTracker(tracker) + + with ( + patch("codecarbon.core.resource_tracker.is_mac_os", return_value=True), + patch("codecarbon.core.resource_tracker.is_linux_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_windows_os", return_value=False), + patch( + "codecarbon.core.resource_tracker.detect_cpu_model", + return_value="Intel(R) Core(TM) i7", + ), + patch("codecarbon.core.resource_tracker.is_mac_arm", return_value=False), + patch( + "codecarbon.core.resource_tracker.cpu.is_powergadget_available", + return_value=True, + ), + patch.object(resource_tracker, "_setup_power_gadget") as mock_power_gadget, + ): + assert resource_tracker._try_platform_cpu_backend() is True + + mock_power_gadget.assert_called_once_with() + + +def test_try_platform_cpu_backend_mac_intel_falls_back_to_powermetrics(): + tracker = make_tracker() + resource_tracker = ResourceTracker(tracker) + + with ( + patch("codecarbon.core.resource_tracker.is_mac_os", return_value=True), + patch("codecarbon.core.resource_tracker.is_linux_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_windows_os", return_value=False), + patch( + "codecarbon.core.resource_tracker.detect_cpu_model", + return_value="Intel(R) Core(TM) i7", + ), + patch("codecarbon.core.resource_tracker.is_mac_arm", return_value=False), + patch( + "codecarbon.core.resource_tracker.cpu.is_powergadget_available", + return_value=False, + ), + patch( + "codecarbon.core.resource_tracker.powermetrics.is_powermetrics_available", + return_value=True, + ), + patch.object(resource_tracker, "_setup_powermetrics") as mock_powermetrics, + ): + assert resource_tracker._try_platform_cpu_backend() is True + + mock_powermetrics.assert_called_once_with() + + +def test_set_cpu_tracking_mac_arm_falls_back_to_powermetrics_when_cpu_load_unavailable(): + tracker = make_tracker() + resource_tracker = ResourceTracker(tracker) + + with ( + patch("codecarbon.core.resource_tracker.is_mac_os", return_value=True), + patch("codecarbon.core.resource_tracker.is_linux_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_windows_os", return_value=False), + patch( + "codecarbon.core.resource_tracker.detect_cpu_model", + return_value="Apple M4", + ), + patch( + "codecarbon.core.resource_tracker.powermetrics.is_powermetrics_available", + return_value=True, + ), + patch.object(resource_tracker, "_setup_cpu_load_fast", return_value=False), + patch.object(resource_tracker, "_setup_powermetrics") as mock_powermetrics, + ): + resource_tracker.set_CPU_tracking() + + mock_powermetrics.assert_called_once_with() + + def test_set_cpu_tracking_prefers_rapl_before_powermetrics(): tracker = make_tracker() resource_tracker = ResourceTracker(tracker) with ( + patch("codecarbon.core.resource_tracker.is_linux_os", return_value=True), + patch("codecarbon.core.resource_tracker.is_mac_os", return_value=False), + patch("codecarbon.core.resource_tracker.is_windows_os", return_value=False), patch( "codecarbon.core.resource_tracker.cpu.is_powergadget_available", return_value=False, @@ -277,7 +403,7 @@ def test_set_cpu_tracking_falls_back_when_forced_power_is_set(): "codecarbon.core.resource_tracker.powermetrics.is_powermetrics_available", return_value=True, ), - patch("codecarbon.core.resource_tracker.cpu.TDP", return_value=fake_tdp), + patch("codecarbon.core.resource_tracker.get_cached_tdp", return_value=fake_tdp), patch.object(resource_tracker, "_setup_fallback_tracking") as mock_fallback, ): resource_tracker.set_CPU_tracking() @@ -350,3 +476,33 @@ def test_set_cpu_gpu_ram_tracking_calls_all_setup_steps(): mock_ram.assert_called_once_with() mock_cpu.assert_called_once_with() mock_gpu.assert_called_once_with() + + +def test_hardware_cache_reuses_setup(): + from codecarbon.core import hardware_cache + + hardware_cache.clear_cache() + key = hardware_cache.make_key(make_tracker()) + hardware_cache._plans[key] = hardware_cache._HardwarePlan( + ram_tracker="cached_ram", + cpu_tracker="cached_cpu", + gpu_tracker="cached_gpu", + conf={"cpu_model": "Cached CPU", "gpu_count": 0, "gpu_model": ""}, + hardware_specs=[], + ) + + tracker2 = make_tracker() + rt2 = ResourceTracker(tracker2) + with ( + patch.object(rt2, "set_RAM_tracking") as mock_ram, + patch.object(rt2, "set_CPU_tracking") as mock_cpu, + patch.object(rt2, "set_GPU_tracking") as mock_gpu, + ): + rt2.set_CPU_GPU_ram_tracking() + mock_ram.assert_not_called() + mock_cpu.assert_not_called() + mock_gpu.assert_not_called() + + assert rt2.cpu_tracker == "cached_cpu" + assert tracker2._conf.get("cpu_model") == "Cached CPU" + hardware_cache.clear_cache()