Adastra

Handle power and energy_accumulator

Adastra

Adastra Doc

Author: benoit-cty

codecarbon/core/gpu.py

@@ -303,20 +303,26 @@ def _get_total_energy_consumption(self):
             energy_count = self._call_amdsmi_with_reinit(
                 amdsmi.amdsmi_get_energy_count, self.handle
             )
+            energy_key = None
+            if "energy_accumulator" in energy_count:
+                energy_key = "energy_accumulator"
+            elif "power" in energy_count:
+                energy_key = "power"
+            if energy_key is None:
+                logger.warning(
+                    f"Neither 'energy_accumulator' nor 'power' found in energy_count: {energy_count}"
+                )
+                return None
             # The amdsmi library returns a dict with energy counter and resolution
             # The counter is the actual accumulated value, resolution tells us how much each unit is worth
-            counter_value = energy_count.get("energy_accumulator", 0)
+            counter_value = energy_count.get(energy_key, 0)
             counter_resolution_uj = energy_count.get("counter_resolution", 0)
             if counter_value == 0 and counter_resolution_uj > 0:
                 # In some cases, the energy_accumulator is 0 but it exist in the metrics info, try to get it from there as a fallback
                 metrics_info = self._get_gpu_metrics_info()
-                counter_value = metrics_info.get("energy_accumulator", 0)
+                counter_value = metrics_info.get(energy_key, 0)
                 logger.debug(
-                    f"Energy accumulator value from metrics info : {counter_value} for GPU {self._gpu_name} with handle {self.handle} {metrics_info=}"
-                )
-
-            if counter_value == 0 or counter_resolution_uj == 0:
-                logger.warning(
+                    f"Energy accumulator value from metrics info : {counter_value} for GPU handle {self.handle} {metrics_info=}"
                     f"Failed to retrieve AMD GPU energy accumulator. energy_count: {energy_count} {counter_value=} {counter_resolution_uj=}",
                     exc_info=True,
                 )

examples/slurm_rocm/README.md

@@ -0,0 +1,182 @@
+# CodeCarbon on CINES Adastra HPC with AMD ROCM
+
+This project was provided with computing and storage resources by GENCI at CINES thanks to the grant AD010615147R1 on the supercomputer Adastra's MI250x/MI300 partition.
+
+Thanks to this grant we were able to develop and test the AMD ROCM support in CodeCarbon, and provide this quick start guide to help other users of Adastra HPC to easily monitor the carbon emissions of their machine learning workloads running on AMD GPUs.
+
+It was tested on Adastra but it will likely work on any SLURM cluster with AMD GPUs and ROCM support.
+
+## Quick Start Guide
+
+Adastra security rules require users to connect through a fixed IP. We choose to setup a small host in the cloud to act as a bastion server, allowing us to connect to Adastra from anywhere without needing to change our IP address.
+
+Adastra architecture is quite standard for a HPC cluster, with a login node and compute nodes. The login node has internet access and is the only one accessible from outside, while the compute nodes are where the GPU workloads run, without internet access.
+
+The Python environment is setup on the login node, and referenced by the compute nodes.
+
+The job is submitted from the login node using `sbatch`, and the SLURM script takes care of loading the Python environment and running the code on the compute node.
+
+If the `--time` option of `sbatch` is less than 30 minutes, the job will be put in the `debug` partition, which has a faster scheduling but a shorter maximum runtime.
+
+### Export your configuration
+
+Adapt the following environment variables with your own configuration. You can add them to your `.bashrc` or `.zshrc` for convenience.
+
+```bash
+export BASTION_IP="xx.xx.xx.xx"
+export BASTION_USER="username"
+export HPC_HOST="xx.xx.fr"
+export HPC_PASS="xxxxx"
+export PROJECT_ID="xxx"
+export USER_NAME="username_hpc"
+export HPC_PROJECT_FOLDER="/lus/home/xxx"
+```
+
+### Connect to CINES Adastra
+
+```bash
+sshpass -p "$HPC_PASS" ssh -J $BASTION_USER@$BASTION_IP $USER_NAME@$HPC_HOST
+```
+
+For the first time you may want to connect one-by-one to debug any SSH issue before using `sshpass`:
+
+```bash
+ssh -o ServerAliveInterval=60 $BASTION_USER@$BASTION_IP
+ssh -o ServerAliveInterval=60 $USER_NAME@$HPC_HOST
+```
+
+### Copy your code to Adastra
+
+```bash
+sshpass -p "$HPC_PASS" scp -r -J $BASTION_USER@$BASTION_IP /you/folder/* $USER_NAME@$HPC_HOST:$HPC_PROJECT_FOLDER
+```
+
+### Install CodeCarbon and dependencies
+
+Be careful to install the correct version of `amdsmi` that is compatible with the ROCM version on Adastra. The last available version we used is `7.0.1`.
+
+#### Simple installation
+
+
+```bash
+module load python/3.12
+module load rocm/7.0.1
+
+python -m venv .venv
+source .venv/bin/activate
+pip install --upgrade pip
+# Important: Adastra's MI250 runs ROCm 6.4.3 natively.
+# With export ROCM_PATH=/opt/rocm-6.4.3 in our SLURM script, this python wheel perfectly matches the C library without symlink issues!
+pip install amdsmi==7.0.1
+pip install codecarbon
+```
+
+#### use a branch of CodeCarbon with PyTorch
+
+```bash
+module load python/3.12
+module load rocm/7.0.1
+git clone https://github.com/mlco2/codecarbon.git
+# If you want a specific version, use git checkout <tag> to switch to the desired version.
+git checkout -b feat/rocm
+cd codecarbon
+python -m venv .venv
+source .venv/bin/activate
+python -V
+# Must be 3.12.x
+pip install --upgrade pip
+# Important: Adastra's MI250 runs ROCm 6.4.3 natively.
+# With export ROCM_PATH=/opt/rocm-6.4.3 in our SLURM script, this python wheel perfectly matches the C library without symlink issues!
+pip install amdsmi==7.0.1
+# Look at https://download.pytorch.org/whl/torch/ for the correct version matching your Python (cp312) and ROCM version.
+# torch-2.10.0+rocm7.0-cp312-cp312-manylinux_2_28_x86_64.whl
+pip3 install torch==2.10.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm7.0
+pip install numpy
+
+# Install CodeCarbon in editable mode to allow for live code changes without reinstallation
+pip install -e .
+```
+
+### Submit a Job
+
+**Option A: Using sbatch (recommended)**
+```bash
+sbatch examples/slurm_rocm/run_codecarbon_pytorch.slurm
+```
+
+### 4. Monitor Job Status
+```bash
+# View running jobs
+squeue -u $USER
+
+# View job output
+tail -f logs/<job_id>.out
+```
+
+## Troubleshooting
+
+
+```
+Error :
+[codecarbon WARNING @ 10:28:46] AMD GPU detected but amdsmi is not properly configured. Please ensure amdsmi is correctly installed to get GPU metrics.Tips : check consistency between Python amdsmi package and ROCm versions, and ensure AMD drivers are up to date. Error: /opt/rocm/lib/libamd_smi.so: undefined symbol: amdsmi_get_cpu_affinity_with_scope
+```
+
+This mean you have a mismatch between the `amdsmi` Python package and the ROCM version installed on Adastra. To fix this, ensure you install the correct version of `amdsmi` that matches the ROCM version (e.g., `amdsmi==7.0.1` for ROCM 7.0.1).
+
+```bash
+KeyError: 'ROCM_PATH'
+```
+This means the rocm module is not loaded, load it with `module load rocm/7.0.1`.
+
+## Limitations and Future Work
+
+The AMD Instinct MI250 accelerator card contains two Graphics Compute Dies (GCDs) per physical card. However, when monitoring energy consumption (e.g., via rocm-smi or tools like CodeCarbon), only one GCD reports power usage, while the other shows zero values. This is problematic for accurate energy accounting, especially in HPC/SLURM environments where jobs may be allocated a single GCD.
+
+So in that case we display a warning.
+
+In a future work we will use `average_gfx_activity` to estimate the corresponding power of both GCDs, and provide an estimation instead of 0.
+
+## Documentation
+
+- [CINES Adastra GPU allocation](https://dci.dci-gitlab.cines.fr/webextranet/user_support/index.html#allocating-a-single-gpu)
+- [CINES PyTorch on ROCM](https://dci.dci-gitlab.cines.fr/webextranet/software_stack/libraries/index.html#pytorch)
+- [AMD SMI library](https://rocm.docs.amd.com/projects/amdsmi/en/latest/reference/amdsmi-py-api.html)
+
+
+## Annex: Example of Job Details with scontrol
+
+This trace was obtained to adapt `codecarbon/core/util.py` to properly parse the SLURM job details and extract the relevant information about GPU and CPU allocation. 
+
+```
+[$PROJECT_ID] $USER_NAME@login5:~/codecarbon$ scontrol show job  4687018
+JobId=4687018 JobName=codecarbon-test
+   UserId=$USER_NAME(xxx) GroupId=grp_$USER_NAME(xxx) MCS_label=N/A
+   Priority=900000 Nice=0 Account=xxxxxx QOS=debug
+   JobState=COMPLETED Reason=None Dependency=(null)
+   Requeue=0 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0
+   RunTime=00:00:24 TimeLimit=00:05:00 TimeMin=N/A
+   SubmitTime=2026-03-02T17:12:49 EligibleTime=2026-03-02T17:12:49
+   AccrueTime=2026-03-02T17:12:49
+   StartTime=2026-03-02T17:12:49 EndTime=2026-03-02T17:13:13 Deadline=N/A
+   SuspendTime=None SecsPreSuspend=0 LastSchedEval=2026-03-02T17:12:49 Scheduler=Main
+   Partition=mi250-shared AllocNode:Sid=login5:2553535
+   ReqNodeList=(null) ExcNodeList=(null)
+   NodeList=g1341
+   BatchHost=g1341
+   NumNodes=1 NumCPUs=16 NumTasks=1 CPUs/Task=8 ReqB:S:C:T=0:0:*:1
+   ReqTRES=cpu=8,mem=29000M,node=1,billing=8,gres/gpu=1
+   AllocTRES=cpu=16,mem=29000M,energy=10211,node=1,billing=16,gres/gpu=1,gres/gpu:mi250x=1
+   Socks/Node=* NtasksPerN:B:S:C=1:0:*:1 CoreSpec=*
+   MinCPUsNode=8 MinMemoryNode=29000M MinTmpDiskNode=0
+   Features=MI250&DEBUG DelayBoot=00:00:00
+   OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null)
+   Command=/lus/home/CT6/$PROJECT_ID/$USER_NAME/codecarbon/run_codecarbon.sh
+   WorkDir=/lus/home/CT6/$PROJECT_ID/$USER_NAME/codecarbon
+   AdminComment=Accounting=1 
+   StdErr=/lus/home/CT6/$PROJECT_ID/$USER_NAME/codecarbon/logs/4687018.err
+   StdIn=/dev/null
+   StdOut=/lus/home/CT6/$PROJECT_ID/$USER_NAME/codecarbon/logs/4687018.out
+   TresPerNode=gres/gpu:1
+   TresPerTask=cpu=8
+```
+

examples/slurm_rocm/amdsmi_demo.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+import amdsmi
+
+
+def main():
+    try:
+        # Initialize AMD SMI
+        amdsmi.amdsmi_init()
+
+        # Get all GPU handles
+        devices = amdsmi.amdsmi_get_processor_handles()
+
+        if not devices:
+            print("No AMD GPUs detected.")
+            return
+
+        for idx, device in enumerate(devices):
+            print(f"\n===== GPU {idx} =====")
+
+            # Get GPU metrics
+            metrics = amdsmi.amdsmi_get_gpu_metrics_info(device)
+
+            # Energy (microjoules)
+            energy = metrics.get("energy_accumulator", None)
+
+            # Power (microwatts)
+            avg_power = metrics.get("average_socket_power", None)
+            cur_power = metrics.get("current_socket_power", None)
+
+            print(f"Energy accumulator      : {energy} uJ")
+            print(f"Average socket power    : {avg_power} W")
+            print(f"Current socket power    : {cur_power} W")
+
+        amdsmi.amdsmi_shut_down()
+
+    except Exception as e:
+        print("Error:", str(e))
+
+
+if __name__ == "__main__":
+    main()

examples/slurm_rocm/amdsmi_demo.slurm

@@ -0,0 +1,55 @@
+#!/bin/bash
+#SBATCH --account=cad15147
+#SBATCH --constraint=MI250
+#SBATCH --nodes=1
+#SBATCH --time=0:25:00
+#SBATCH --gpus-per-node=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=8
+#SBATCH --threads-per-core=1
+#SBATCH --job-name=codecarbon-test
+#SBATCH --output=logs/%j.out
+#SBATCH --error=logs/%j.err
+
+# Load AMD ROCM environment
+module purge
+module load cpe/24.07
+module load python/3.12
+module load rocm/7.0.1
+
+# Print environment info
+echo "=== Job Environment ==="
+echo "Running on host: $(hostname)"
+echo "at: $(date)"
+echo "Job ID: $SLURM_JOB_ID"
+echo "Number of GPUs: $SLURM_GPUS_PER_NODE"
+echo "ROCR_VISIBLE_DEVICES: $ROCR_VISIBLE_DEVICES"
+echo "HSA_OVERRIDE_GFX_VERSION: $HSA_OVERRIDE_GFX_VERSION"
+echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
+echo "PATH: $PATH"
+export PYTHONPATH=/opt/rocm-7.0.1/share/amd_smi:$PYTHONPATH
+echo "PYTHONPATH: $PYTHONPATH"
+
+rocm-smi
+rocm-smi --version
+rocm-smi --showmetrics --json
+
+
+# Create logs directory if it doesn't exist
+mkdir -p logs
+
+# Run the Python script
+pip install amdsmi==7.0.1
+echo "=== Installed AMD SMI Python Package ==="
+python3 -m pip list | grep -E "(amd)"
+echo "=== AMD SMI Metrics ==="
+amd-smi -h
+# Verify activation (optional)
+echo "=== Python Version ==="
+which python3
+python3 --version
+ls /opt/rocm-7.0.1/share/amd_smi
+echo "=== ls /opt ==="
+ls /opt
+echo "=== Running Training Script ==="
+srun python amdsmi_demo.py

examples/slurm_rocm/no_load.py

@@ -0,0 +1,24 @@
+"""
+Use CodeCarbon but without loading the AMD GPU.
+pip install codecarbon
+"""
+
+import time
+
+from codecarbon import track_emissions
+
+
+@track_emissions(
+    measure_power_secs=5,
+    log_level="debug",
+)
+def train_model():
+    """
+    This function will do nothing.
+    """
+    print("10 seconds before ending script...")
+    time.sleep(10)
+
+
+if __name__ == "__main__":
+    model = train_model()