Build: Bump luigi-tools and fix results with Pandas>=3

Author: adrien-berchet

data_validation_framework/util.py

@@ -73,6 +73,15 @@ def _apply_to_df_internal(data):
     return num, df.apply(_tqdm_wrapper, axis=1, args=args, **kwargs)
 
 
+def _restore_object_nulls(result_df, template_df):
+    """Restore None values for columns that were object-typed in the input."""
+    for col, dtype in template_df.dtypes.items():
+        if col in result_df.columns and pd.api.types.is_object_dtype(dtype):
+            result_df[col] = result_df[col].astype(object)
+            result_df.loc[result_df[col].isna(), col] = None
+    return result_df
+
+
 def tqdm_worker(progress_bar, tqdm_queue):
     """Update progress bar using the Queue."""
     while True:
@@ -155,7 +164,7 @@ def apply_to_df(df, func, *args, nb_processes=None, redirect_stdout=None, **kwar
     progress_bar.close()
 
     # Return the results
-    return all_res
+    return _restore_object_nulls(all_res, df)
 
 
 def try_operation(row, func, *args, **kwargs):

setup.py

@@ -7,7 +7,7 @@
 
 reqs = [
     "luigi>=3.1",
-    "luigi-tools>=0.0.18",
+    "luigi-tools>=0.3.5",
     "numpy>=1.21",
     "pandas>=1.3",
     "rst2pdf>=0.99",

tests/test_task.py

@@ -31,6 +31,8 @@
 
 SKIP_IF_NO_LATEXMK = not which("latexmk")
 REASON_NO_LATEXMK = "The command latexmk is not available."
+# Newer Sphinx/TeX stacks slightly shift pagination while keeping the report content intact.
+LATEXPDF_DIFF_THRESHOLD = 15
 
 
 @pytest.fixture
@@ -2107,7 +2109,9 @@ def test_latexpdf(
             assert (root / "TestWorkflow" / "report.csv").exists()
             assert (root / "report_TestWorkflow.pdf").exists()
             assert pdfdiff(
-                root / "report_TestWorkflow.pdf", data_dir / "test_report" / "report_latexpdf.pdf"
+                root / "report_TestWorkflow.pdf",
+                data_dir / "test_report" / "report_latexpdf.pdf",
+                threshold=LATEXPDF_DIFF_THRESHOLD,
             )
 
         def test_fail_element_no_exception(
@@ -2443,7 +2447,7 @@ def test_latexpdf(self, tmpdir, dataset_df_path, data_dir, TestWorkflow):
             assert pdfdiff(
                 root / "TestWorkflow_specifications.pdf",
                 data_dir / "test_report_before_run" / "report_latexpdf.pdf",
-                threshold=15,
+                threshold=LATEXPDF_DIFF_THRESHOLD,
             )
 
         @pytest.fixture
@@ -2502,6 +2506,7 @@ def test_latexpdf_with_config(
             assert pdfdiff(
                 root / "TestWorkflow_specifications.pdf",
                 data_dir / "test_report_before_run" / "report_latexpdf_with_config.pdf",
+                threshold=LATEXPDF_DIFF_THRESHOLD,
             )
 
         def test_nested_workflows(

tox.ini

@@ -56,14 +56,17 @@ commands =
 basepython = python3.9
 deps =
     uv
+allowlist_externals =
+    uv
 commands_pre =
     uv pip compile \
+        --python {envpython} \
         --resolution lowest-direct \
         --extra test \
         --output-file {envtmpdir}/requirements_min.txt \
         setup.py
-    pip install -r {envtmpdir}/requirements_min.txt
-    pip freeze
+    uv pip install --python {envpython} -r {envtmpdir}/requirements_min.txt
+    uv pip freeze --python {envpython}
 
 [testenv:lint]
 basepython = python3.9
@@ -85,7 +88,7 @@ commands =
     pre-commit run --all-files
 
 [testenv:docs]
-basepython = python3.9
+basepython = python3.11
 changedir = docs
 extras = docs
 allowlist_externals =