Fix unwanted aggregation of duplicate records

in `build_dataframe_from_csv` function

Solves #1323

Author: MariusWirtz

TM1py/Utils/Utils.py

@@ -617,14 +617,23 @@ def build_dataframe_from_csv(
         return df
 
     # due to csv creation logic, last column is bottom dimension from the column selection
-    df = df.pivot_table(
-        index=tuple(df.columns[:-2]),
-        aggfunc="sum",
-        columns=df.columns[-2],
-        values=df.columns[-1],
-        dropna=True,
-        sort=False,
-    ).reset_index()
+    idx_cols = list(df.columns[:-2])
+    col_col = df.columns[-2]
+    val_col = df.columns[-1]
+
+    # create a unique row id within each (idx_cols + col_col) group
+    df["_dup"] = df.groupby(idx_cols + [col_col]).cumcount()
+
+    df = (
+        df.pivot(
+            index=idx_cols + ["_dup"],
+            columns=col_col,
+            values=val_col,
+        )
+        .dropna(how="all")
+        .reset_index()
+        .drop(columns="_dup")
+    )
 
     # drop title on index
     return df.rename_axis(None, axis=1)

Tests/Utils_test.py

@@ -150,6 +150,34 @@ def test_build_dataframe_from_csv(self):
 
         pd._testing.assert_frame_equal(expected_df, df, check_column_type=False)
 
+    def test_build_dataframe_from_csv_shaped(self):
+        raw_csv = "Region~Product~Measure~Value\r\n" "r1~p1~Revenue~1.0\r\n" "r1~p2~Revenue~3.0\r\n" "r2~p2~Revenue~4.0"
+        df = build_dataframe_from_csv(raw_csv, dtype={"Revenue": float}, shaped=True)
+
+        expected_df = pd.DataFrame(
+            {
+                "Region":  ["r1", "r1", "r2"],
+                "Product": ["p1", "p2", "p2"],
+                "Revenue": [1.00000, 3.00000, 4.00000],
+            }
+        )
+
+        pd._testing.assert_frame_equal(expected_df, df, check_column_type=False)
+
+    def test_build_dataframe_from_csv_shaped_with_duplicates(self):
+        raw_csv = "Region~Product~Measure~Value\r\n" "r1~p1~Revenue~1.0\r\n" "r1~p1~Revenue~1.0\r\n" "r2~p2~Revenue~4.0"
+        df = build_dataframe_from_csv(raw_csv, dtype={"Revenue": float}, shaped=True)
+
+        expected_df = pd.DataFrame(
+            {
+                "Region":  ["r1", "r1", "r2"],
+                "Product": ["p1", "p1", "p2"],
+                "Revenue": [1.00000, 1.00000, 4.00000],
+            }
+        )
+
+        pd._testing.assert_frame_equal(expected_df, df, check_column_type=False)
+
     def test_build_dataframe_from_csv_with_none_element_and_none_value(self):
         raw_csv = "d1~d2~Value\r\n" "e1~e1~None\r\n" "e1~e2~2.0\r\n" "None~e1~3.0\r\n" "None~e2~4.0"
         df = build_dataframe_from_csv(raw_csv)