Add boundary condition tests for HTML formatter memory limits and resolve max_rows logic

Author: kosiew

python/tests/test_dataframe.py

@@ -1458,6 +1458,55 @@ def test_html_formatter_memory(df, clean_formatter_state):
     assert "data truncated" not in html_output.lower()
 
 
+def test_html_formatter_memory_boundary_conditions(df, clean_formatter_state):
+    """Test memory limit behavior at boundary conditions.
+    
+    This test validates that the formatter correctly handles edge cases when
+    the memory limit is very close to actual data size, ensuring that min_rows
+    constraint is properly respected while respecting memory limits.
+    """
+    # Get the raw size of the data to test boundary conditions
+    # First, capture output with no limits
+    configure_formatter(max_memory_bytes=10 * MB, min_rows_display=1, max_rows=100)
+    unrestricted_output = df._repr_html_()
+    unrestricted_rows = count_table_rows(unrestricted_output)
+
+    # Test 1: Very small memory limit should still respect min_rows
+    configure_formatter(max_memory_bytes=10, min_rows_display=1)
+    html_output = df._repr_html_()
+    tr_count = count_table_rows(html_output)
+    assert tr_count >= 2  # At least header + 1 data row (minimum)
+    # Should show truncation since we limited memory so aggressively
+    assert "data truncated" in html_output.lower()
+
+    # Test 2: Memory limit at default size should work well
+    configure_formatter(max_memory_bytes=2 * MB, min_rows_display=1)
+    html_output = df._repr_html_()
+    tr_count = count_table_rows(html_output)
+    assert tr_count >= 2  # At least header + min_rows
+
+    # Test 3: Very large memory limit should show all data
+    configure_formatter(max_memory_bytes=100 * MB, min_rows_display=1)
+    html_output = df._repr_html_()
+    tr_count = count_table_rows(html_output)
+    assert tr_count == unrestricted_rows  # Should show all rows
+
+    # Test 4: Min rows should override memory limit
+    # With tiny memory and larger min_rows, min_rows should win
+    configure_formatter(max_memory_bytes=10, min_rows_display=2)
+    html_output = df._repr_html_()
+    tr_count = count_table_rows(html_output)
+    assert tr_count >= 3  # At least header + 2 data rows (min_rows)
+    # Should show truncation message despite min_rows being satisfied
+    assert "data truncated" in html_output.lower()
+
+    # Test 5: Default memory limit with different min_rows
+    configure_formatter(max_memory_bytes=2 * MB, min_rows_display=2, max_rows=2)
+    html_output = df._repr_html_()
+    tr_count = count_table_rows(html_output)
+    assert tr_count == 3  # header + 2 data rows
+
+
 def test_html_formatter_max_rows(df, clean_formatter_state):
     configure_formatter(min_rows_display=2, max_rows=2)
     html_output = df._repr_html_()

src/dataframe.rs

@@ -148,18 +148,31 @@ where
         .unwrap_or_else(|_| default_value.clone())
 }
 
+/// Resolve the max_rows value, preferring repr_rows if it differs from the default.
+///
+/// This function handles the transition from the deprecated `repr_rows` parameter
+/// to the new `max_rows` parameter. It checks both attributes and uses `repr_rows`
+/// if it has been explicitly set to a different value than `max_rows`.
+fn resolve_max_rows(formatter: &Bound<'_, PyAny>, default: usize) -> usize {
+    let max_rows = get_attr(formatter, "max_rows", default);
+    let repr_rows = get_attr(formatter, "repr_rows", default);
+
+    // If repr_rows differs from the default, it was explicitly set by the user
+    // (Python-side validation ensures only one is used, but we prefer repr_rows
+    // for backward compatibility in case it was set)
+    if repr_rows != default && repr_rows != max_rows {
+        repr_rows
+    } else {
+        max_rows
+    }
+}
+
 /// Helper function to create a FormatterConfig from a Python formatter object
 fn build_formatter_config_from_python(formatter: &Bound<'_, PyAny>) -> PyResult<FormatterConfig> {
     let default_config = FormatterConfig::default();
     let max_bytes = get_attr(formatter, "max_memory_bytes", default_config.max_bytes);
     let min_rows = get_attr(formatter, "min_rows_display", default_config.min_rows);
-    let max_rows = get_attr(formatter, "max_rows", default_config.max_rows);
-    let repr_rows = get_attr(formatter, "repr_rows", max_rows);
-    let max_rows = if repr_rows != max_rows {
-        repr_rows
-    } else {
-        max_rows
-    };
+    let max_rows = resolve_max_rows(formatter, default_config.max_rows);
 
     let config = FormatterConfig {
         max_bytes,
@@ -1360,7 +1373,10 @@ async fn collect_record_batches_to_display(
     let mut record_batches = Vec::default();
     let mut has_more = false;
 
-    // ensure minimum rows even if memory/row limits are hit
+    // Collect rows until we hit a limit (memory or max_rows) OR reach the guaranteed minimum.
+    // The minimum rows constraint overrides both memory and row limits to ensure a baseline
+    // of data is always displayed, even if it temporarily exceeds those limits.
+    // This provides better UX by guaranteeing users see at least min_rows rows.
     while (size_estimate_so_far < max_bytes && rows_so_far < max_rows) || rows_so_far < min_rows {
         let mut rb = match stream.next().await {
             None => {
@@ -1374,11 +1390,14 @@ async fn collect_record_batches_to_display(
         if rows_in_rb > 0 {
             size_estimate_so_far += rb.get_array_memory_size();
 
+            // When memory limit is exceeded, scale back row count proportionally to stay within budget
             if size_estimate_so_far > max_bytes {
                 let ratio = max_bytes as f32 / size_estimate_so_far as f32;
                 let total_rows = rows_in_rb + rows_so_far;
 
+                // Calculate reduced rows maintaining the memory/data proportion
                 let mut reduced_row_num = (total_rows as f32 * ratio).round() as usize;
+                // Ensure we always respect the minimum rows guarantee
                 if reduced_row_num < min_rows {
                     reduced_row_num = min_rows.min(total_rows);
                 }