Cg00014 (#1281)

* updated to_dict dask

* update dask

* fix

Author: SFJohnson24

cdisc_rules_engine/models/actions.py

@@ -309,9 +309,13 @@ def _create_error_object(
                 else ""
             ),
             row=(
-                int(source_row_number[df_row.name])
-                if isinstance(source_row_number, pd.Series)
-                else (int(df_row.name) + 1)
+                int(data.loc[df_row.name]["row_number"])
+                if "row_number" in data.columns
+                else (
+                    int(source_row_number[df_row.name])
+                    if isinstance(source_row_number, pd.Series)
+                    else (int(df_row.name) + 1)
+                )
             ),  # record number should start at 1, not 0
             value=filtered_dict,
             usubjid=(

cdisc_rules_engine/models/dataset/dask_dataset.py

@@ -360,7 +360,18 @@ def fillna(
             return self.__class__(result)
 
     def to_dict(self, **kwargs) -> dict:
-        return self._data.compute().to_dict(**kwargs)
+        orient = kwargs.get("orient", "dict")
+        if orient == "records":
+            reset_df = self._data.reset_index(drop=True)
+            all_partitions = list(
+                reset_df.map_partitions(lambda x: x.to_dict(orient="records"))
+            )
+            flattened = []
+            for partition in all_partitions:
+                flattened.extend(partition)
+            return flattened
+        else:
+            return self._data.compute().to_dict(**kwargs)
 
     def items(self, **kwargs):
         computed_df = self._data.compute()

cdisc_rules_engine/services/reporting/base_report.py

@@ -149,6 +149,7 @@ def process_values(self, values: List[str], excel: bool) -> Union[str, List[str]
         for value in values:
             if value is None:
                 processed_values.append("null")
+                continue
             value = value.strip()
             if value == "" or value.lower() == "nan":
                 processed_values.append("null")