feat: store CPU, GPU and RAM tracking methods in EmissionsData

codecarbon/core/hardware_cache.py

@@ -28,6 +28,9 @@
     "gpu_count",
     "gpu_model",
     "gpu_ids",
+    "ram_tracking_method",
+    "cpu_tracking_method",
+    "gpu_tracking_method",
 )
 
 

codecarbon/core/resource_tracker.py

@@ -317,6 +317,10 @@ def _run_full_hardware_setup(self) -> None:
             cpu_future.result()
             gpu_future.result()
 
+        self.tracker._conf["ram_tracking_method"] = self.ram_tracker
+        self.tracker._conf["cpu_tracking_method"] = self.cpu_tracker
+        self.tracker._conf["gpu_tracking_method"] = self.gpu_tracker
+
         logger.info(f"""The below tracking methods have been set up:
                 RAM Tracking Method: {self.ram_tracker}
                 CPU Tracking Method: {self.cpu_tracker}

codecarbon/emissions_tracker.py

@@ -1096,6 +1096,9 @@ def _prepare_emissions_data(self) -> EmissionsData:
             latitude=self._conf.get("latitude"),
             ram_total_size=self._conf.get("ram_total_size"),
             tracking_mode=self._conf.get("tracking_mode"),
+            ram_tracking_method=self._conf.get("ram_tracking_method", ""),
+            cpu_tracking_method=self._conf.get("cpu_tracking_method", ""),
+            gpu_tracking_method=self._conf.get("gpu_tracking_method", ""),
             pue=self._pue,
             wue=self._wue,
         )

codecarbon/output_methods/boamps/mapper.py

@@ -113,13 +113,11 @@ def _build_header(
 
 def _build_measure(emissions: EmissionsData) -> BoAmpsMeasure:
     """Build a BoAmps measure from EmissionsData."""
-    # Note: emissions.tracking_mode is "process"/"machine" (CodeCarbon's scope),
-    # not the CPU/GPU power tracking method (rapl, nvml, etc.) that BoAmps expects
-    # for cpuTrackingMode/gpuTrackingMode. We omit these fields since we don't
-    # have the actual tracker implementation details in EmissionsData.
     measure = BoAmpsMeasure(
         measurement_method="codecarbon",
         version=emissions.codecarbon_version,
+        cpu_tracking_mode=emissions.cpu_tracking_method or None,
+        gpu_tracking_mode=emissions.gpu_tracking_method or None,
         power_consumption=emissions.energy_consumed,
         measurement_duration=emissions.duration,
         measurement_date_time=_to_boamps_datetime(emissions.timestamp),

codecarbon/output_methods/emissions_data.py

@@ -47,6 +47,9 @@ class EmissionsData:
     on_cloud: str = "N"
     pue: float = 1
     wue: float = 0
+    ram_tracking_method: str = ""
+    cpu_tracking_method: str = ""
+    gpu_tracking_method: str = ""
 
     @property
     def values(self) -> OrderedDict:

docs/reference/output.md

@@ -64,6 +64,9 @@ The package has an in-built logger that logs data into a CSV file named `emissio
 | gpu_utilization_percent | Average GPU utilization during tracking period (%) |
 | ram_utilization_percent | Average RAM utilization during tracking period (%) |
 | ram_used_gb | Average RAM used during tracking period (GB) |
+| ram_tracking_method | How RAM power was obtained (e.g., `RAM power estimation model`) |
+| cpu_tracking_method | How CPU power was obtained (e.g., `RAPL`, `cpu_load`, `TDP constant`) |
+| gpu_tracking_method | How GPU power was obtained (e.g., `pynvml`, `amdsmi`, `Unspecified`) |
 
 !!! note
     Developers can enhance the Output interface by implementing a custom class that extends `BaseOutput` at `codecarbon/output.py`. For example, to log into a database.

tests/output_methods/test_file.py

@@ -48,6 +48,9 @@ def setUp(self):
             on_cloud="true",
             pue=1.5,
             wue=0.5,
+            ram_tracking_method="RAM power estimation model",
+            cpu_tracking_method="RAPL",
+            gpu_tracking_method="pynvml",
         )
 
     def tearDown(self):

tests/test_boamps_output.py

@@ -336,8 +336,18 @@ def test_gpu_utilization_as_fraction(self):
         measure = report.measures[0].to_dict()
         self.assertAlmostEqual(measure["averageUtilizationGpu"], 0.80, places=2)
 
-    def test_tracking_modes_omitted(self):
-        """cpuTrackingMode/gpuTrackingMode are omitted (not available in EmissionsData)."""
+    def test_tracking_modes_mapped(self):
+        """cpuTrackingMode/gpuTrackingMode come from the tracking methods."""
+        emissions = _make_emissions_data(
+            cpu_tracking_method="RAPL", gpu_tracking_method="pynvml"
+        )
+        report = map_emissions_to_boamps(emissions, task=self.task)
+        measure = report.measures[0].to_dict()
+        self.assertEqual(measure["cpuTrackingMode"], "RAPL")
+        self.assertEqual(measure["gpuTrackingMode"], "pynvml")
+
+    def test_tracking_modes_omitted_when_unknown(self):
+        """cpuTrackingMode/gpuTrackingMode are omitted when no method was recorded."""
         report = map_emissions_to_boamps(self.emissions, task=self.task)
         measure = report.measures[0].to_dict()
         self.assertNotIn("cpuTrackingMode", measure)

tests/test_data/emissions_valid_headers.csv

@@ -1,2 +1,2 @@
-timestamp,project_name,run_id,experiment_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,water_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,cpu_utilization_percent,gpu_utilization_percent,ram_utilization_percent,ram_used_gb,on_cloud,pue,wue
-2021-09-23T15:04:51,codecarbon,0a578547-1d6b-4e2f-be0c-7ad10f2f7c97,test,161.20380687713623,0.0004490989249167,0.0027859076880178,0.269999999999999,0.0,12.884901888000002,0.0,0,0.00057442898176,0.00057442898176,0.1,Morocco,MAR,casablanca-settat,,,macOS-10.15.7-x86_64-i386-64bit,3.8.0,2.1.3,12,Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz,,,-7.9084,33.5932,,machine,0.0,0.0,0.0,0.0,N,1.0,0.0
+timestamp,project_name,run_id,experiment_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,water_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,cpu_utilization_percent,gpu_utilization_percent,ram_utilization_percent,ram_used_gb,on_cloud,pue,wue,ram_tracking_method,cpu_tracking_method,gpu_tracking_method
+2021-09-23T15:04:51,codecarbon,0a578547-1d6b-4e2f-be0c-7ad10f2f7c97,test,161.20380687713623,0.0004490989249167,0.0027859076880178,0.269999999999999,0.0,12.884901888000002,0.0,0,0.00057442898176,0.00057442898176,0.1,Morocco,MAR,casablanca-settat,,,macOS-10.15.7-x86_64-i386-64bit,3.8.0,2.1.3,12,Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz,,,-7.9084,33.5932,,machine,0.0,0.0,0.0,0.0,N,1.0,0.0,RAM power estimation model,cpu_load,Unspecified

tests/test_emissions_tracker.py

@@ -633,6 +633,35 @@ def test_offline_tracker_country_name(
         self.assertEqual("United States", emissions_df["country_name"].values[0])
         self.assertEqual("USA", emissions_df["country_iso_code"].values[0])
 
+    def test_offline_tracker_writes_tracking_methods(
+        self,
+        mock_cli_setup,
+        mock_log_values,
+        mocked_get_gpu_details,
+        mocked_env_cloud_details,
+        mocked_get_gpu_utilization_list,
+        mocked_is_gpu_details_available,
+        mocked_is_nvidia_system,
+    ):
+        tracker = OfflineEmissionsTracker(
+            country_iso_code="USA",
+            output_dir=self.temp_path,
+            experiment_id="test",
+        )
+        tracker.start()
+        heavy_computation(run_time_secs=1)
+        tracker.stop()
+
+        emissions_df = pd.read_csv(self.emissions_file_path)
+
+        for column in (
+            "ram_tracking_method",
+            "cpu_tracking_method",
+            "gpu_tracking_method",
+        ):
+            self.assertIn(column, emissions_df.columns)
+            self.assertTrue(emissions_df[column].values[0])
+
     def test_offline_tracker_invalid_headers(
         self,
         mock_cli_setup,

tests/test_hardware_cache.py

@@ -212,6 +212,36 @@ def test_capture_and_apply_restore_hardware_plan():
     assert type(tracker2._hardware[0]).__name__ == "RAM"
 
 
+def test_capture_and_apply_restore_tracking_methods():
+    tracker = make_tracker(
+        _conf={
+            "ram_tracking_method": "RAM power estimation model",
+            "cpu_tracking_method": "RAPL",
+            "gpu_tracking_method": "pynvml",
+        },
+    )
+    resource_tracker = SimpleNamespace(
+        tracker=tracker,
+        ram_tracker="RAM power estimation model",
+        cpu_tracker="RAPL",
+        gpu_tracker="pynvml",
+    )
+    plan = hardware_cache.capture(resource_tracker)
+
+    tracker2 = make_tracker()
+    rt2 = SimpleNamespace(
+        tracker=tracker2,
+        ram_tracker="Unspecified",
+        cpu_tracker="Unspecified",
+        gpu_tracker="Unspecified",
+    )
+    hardware_cache.apply(rt2, plan)
+
+    assert tracker2._conf["ram_tracking_method"] == "RAM power estimation model"
+    assert tracker2._conf["cpu_tracking_method"] == "RAPL"
+    assert tracker2._conf["gpu_tracking_method"] == "pynvml"
+
+
 def test_get_or_run_setup_runs_setup_once():
     tracker = make_tracker()
     resource_tracker = SimpleNamespace(

tests/test_resource_tracker.py

@@ -507,6 +507,31 @@ def test_set_cpu_gpu_ram_tracking_calls_all_setup_steps():
     mock_gpu.assert_called_once_with()
 
 
+def test_run_full_hardware_setup_stores_tracking_methods_in_conf():
+    tracker = make_tracker()
+    resource_tracker = ResourceTracker(tracker)
+
+    def set_ram():
+        resource_tracker.ram_tracker = "RAM power estimation model"
+
+    def set_cpu():
+        resource_tracker.cpu_tracker = MODE_CPU_LOAD
+
+    def set_gpu():
+        resource_tracker.gpu_tracker = "pynvml"
+
+    with (
+        patch.object(resource_tracker, "set_RAM_tracking", side_effect=set_ram),
+        patch.object(resource_tracker, "set_CPU_tracking", side_effect=set_cpu),
+        patch.object(resource_tracker, "set_GPU_tracking", side_effect=set_gpu),
+    ):
+        resource_tracker._run_full_hardware_setup()
+
+    assert tracker._conf["ram_tracking_method"] == "RAM power estimation model"
+    assert tracker._conf["cpu_tracking_method"] == MODE_CPU_LOAD
+    assert tracker._conf["gpu_tracking_method"] == "pynvml"
+
+
 def test_hardware_cache_reuses_setup():
     from codecarbon.core import hardware_cache