Sort CSV headers before comparing so that we don't spuriously create backup output files.

Author: cianc

codecarbon/output_methods/file.py

@@ -68,10 +68,10 @@ def has_valid_headers(self, data: EmissionsData) -> bool:
                 # No entries
                 return True
             dict_from_csv = dict(csv_entries_list[0])
-            list_of_column_names = list(dict_from_csv.keys())
-            return list(data.values.keys()) == list_of_column_names
+            list_of_column_names = sorted(dict_from_csv.keys())
+            return sorted(data.values.keys()) == list_of_column_names
 
-    def out(self, total: EmissionsData, _: EmissionsData):
+    def out(self, total: EmissionsData, _):
         """
         Save the emissions data from a whole run to a CSV file.
 

tests/output_methods/test_file.py

@@ -2,7 +2,7 @@
 import shutil
 import tempfile
 import unittest
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pandas as pd
 
@@ -66,7 +66,7 @@ def test_file_output_initialization_invalid_dir(self):
 
     def test_has_valid_headers_success(self):
         file_output = FileOutput("test.csv", self.temp_dir)
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         self.assertTrue(file_output.has_valid_headers(self.emissions_data))
 
@@ -77,9 +77,19 @@ def test_has_valid_headers_success_with_empty_file(self):
 
         self.assertTrue(file_output.has_valid_headers(self.emissions_data))
 
+    def test_has_valid_headers_different_order_success(self):
+        file_output = FileOutput("test.csv", self.temp_dir)
+        file_output.out(self.emissions_data, None)
+
+        df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
+        df = df[list(reversed(df.columns))]
+        df.to_csv(os.path.join(self.temp_dir, "test.csv"), index=False)
+
+        self.assertTrue(file_output.has_valid_headers(self.emissions_data))
+
     def test_has_valid_headers_failure(self):
         file_output = FileOutput("test.csv", self.temp_dir)
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         df.rename(columns={"wue": "new_header"}, inplace=True)
@@ -90,10 +100,10 @@ def test_has_valid_headers_failure(self):
     @patch("codecarbon.output_methods.file.FileOutput.has_valid_headers")
     def test_file_output_out_file_exists_invalid_headers(self, mock_has_valid_headers):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append")
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         mock_has_valid_headers.return_value = False
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv.bak"))
         self.assertEqual(len(df), 1)
@@ -102,63 +112,67 @@ def test_file_output_out_file_exists_invalid_headers(self, mock_has_valid_header
 
     def test_file_output_out_update_no_file_exists(self):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update")
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(len(df), 1)
 
     def test_file_output_out_append_no_file_exists(self):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append")
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(len(df), 1)
 
     def test_file_output_out_append_file_exists(self):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append")
-        file_output.out(self.emissions_data, MagicMock())
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
+        file_output.out(self.emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(len(df), 2)
 
     def test_file_output_out_update_file_exists_no_matching_row(self):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update")
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         updated_emissions_data = self.emissions_data
         updated_emissions_data.run_id = "new_test_run_id"
-        file_output.out(updated_emissions_data, MagicMock())
+        file_output.out(updated_emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(len(df), 2)
 
     def test_file_output_out_update_file_exists_multiple_matching_rows(self):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update")
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         # Manually add a duplicate row to simulate the condition
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         df = pd.concat([df, df])
         df.to_csv(os.path.join(self.temp_dir, "test.csv"), index=False)
 
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
 
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(len(df), 3)
 
     def test_file_output_out_update_file_exists_one_matchingrows(self):
         file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update")
-        file_output.out(self.emissions_data, MagicMock())
+        file_output.out(self.emissions_data, None)
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(df["cpu_power"].iloc[0], 20)
 
         new_data = self.emissions_data
         new_data.cpu_power = 2
-        file_output.out(new_data, MagicMock())
+        file_output.out(new_data, None)
         df = pd.read_csv(os.path.join(self.temp_dir, "test.csv"))
         self.assertEqual(df["cpu_power"].iloc[0], 2)
 
+    # def test_file_output_out_consistent_column_ordering(self):
+    #     file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append")
+    #     file_output.out(self.emissions_data, None)
+
     def test_file_output_task_out(self):
         task_emissions_data = [
             TaskEmissionsData(