perf: Optimize array_positions() for scalar needle (#20770)

## Which issue does this PR close?

- Closes #20769.

## Rationale for this change

`array_positions` previously compared the needle against each row's
sub-array individually. When the needle is a scalar (the common case),
we can do a single bulk `arrow_ord::cmp::not_distinct` comparison
against the entire flat values buffer and then walk the result bitmap,
which is significantly faster: the speedup on the `array_positions()`
microbenchmarks ranges from 5x to 40x, depending on the size of the
array.

The same pattern has already been applied to `array_position` (#20532),
and previously to other array UDFs.

## What changes are included in this PR?

- Add benchmarks for `array_positions`.
- Implement bulk-comparison optimization
- Refactor `array_position`'s existing fast path slightly for
consistency
- Code cleanup to use "haystack" and "needle" consistently, not vague
terms like "list_array" and "element"
- Add unit tests for `array_positions` with sliced ListArrays, for peace
of mind
- Add unit tests for sliced lists and sliced lists with nulls for the
new `array_positions` fast path.

## Are these changes tested?

Yes.

## Are there any user-facing changes?

No.

## AI usage

Multiple AI tools were used to iterate on this PR. I have reviewed and
understand the resulting code.

---------

Co-authored-by: Oleks V <comphead@users.noreply.github.com>

Author: neilconway

datafusion/functions-nested/benches/array_position.rs

@@ -24,7 +24,7 @@ use criterion::{
 use datafusion_common::ScalarValue;
 use datafusion_common::config::ConfigOptions;
 use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl};
-use datafusion_functions_nested::position::ArrayPosition;
+use datafusion_functions_nested::position::{ArrayPosition, ArrayPositions};
 use rand::Rng;
 use rand::SeedableRng;
 use rand::rngs::StdRng;
@@ -39,6 +39,7 @@ const SENTINEL_NEEDLE: i64 = -1;
 fn criterion_benchmark(c: &mut Criterion) {
     for size in [10, 100, 500] {
         bench_array_position(c, size);
+        bench_array_positions(c, size);
     }
 }
 
@@ -146,6 +147,112 @@ fn bench_array_position(c: &mut Criterion, array_size: usize) {
     group.finish();
 }
 
+fn bench_array_positions(c: &mut Criterion, array_size: usize) {
+    let mut group = c.benchmark_group("array_positions_i64");
+    let haystack_found_once = create_haystack_with_sentinel(
+        NUM_ROWS,
+        array_size,
+        NULL_DENSITY,
+        SENTINEL_NEEDLE,
+        0,
+    );
+    let haystack_found_many = create_haystack_with_sentinels(
+        NUM_ROWS,
+        array_size,
+        NULL_DENSITY,
+        SENTINEL_NEEDLE,
+    );
+    let haystack_not_found =
+        create_haystack_without_sentinel(NUM_ROWS, array_size, NULL_DENSITY);
+    let num_rows = haystack_not_found.len();
+    let arg_fields: Vec<Arc<Field>> = vec![
+        Field::new("haystack", haystack_not_found.data_type().clone(), false).into(),
+        Field::new("needle", DataType::Int64, false).into(),
+    ];
+    let return_field: Arc<Field> = Field::new(
+        "result",
+        DataType::List(Arc::new(Field::new_list_field(DataType::UInt64, true))),
+        true,
+    )
+    .into();
+    let config_options = Arc::new(ConfigOptions::default());
+    let needle = ScalarValue::Int64(Some(SENTINEL_NEEDLE));
+
+    let args_found_once = vec![
+        ColumnarValue::Array(haystack_found_once.clone()),
+        ColumnarValue::Scalar(needle.clone()),
+    ];
+    group.bench_with_input(
+        BenchmarkId::new("found_once", array_size),
+        &array_size,
+        |b, _| {
+            let udf = ArrayPositions::new();
+            b.iter(|| {
+                black_box(
+                    udf.invoke_with_args(ScalarFunctionArgs {
+                        args: args_found_once.clone(),
+                        arg_fields: arg_fields.clone(),
+                        number_rows: num_rows,
+                        return_field: return_field.clone(),
+                        config_options: config_options.clone(),
+                    })
+                    .unwrap(),
+                )
+            })
+        },
+    );
+
+    let args_found_many = vec![
+        ColumnarValue::Array(haystack_found_many.clone()),
+        ColumnarValue::Scalar(needle.clone()),
+    ];
+    group.bench_with_input(
+        BenchmarkId::new("found_many", array_size),
+        &array_size,
+        |b, _| {
+            let udf = ArrayPositions::new();
+            b.iter(|| {
+                black_box(
+                    udf.invoke_with_args(ScalarFunctionArgs {
+                        args: args_found_many.clone(),
+                        arg_fields: arg_fields.clone(),
+                        number_rows: num_rows,
+                        return_field: return_field.clone(),
+                        config_options: config_options.clone(),
+                    })
+                    .unwrap(),
+                )
+            })
+        },
+    );
+
+    let args_not_found = vec![
+        ColumnarValue::Array(haystack_not_found.clone()),
+        ColumnarValue::Scalar(needle.clone()),
+    ];
+    group.bench_with_input(
+        BenchmarkId::new("not_found", array_size),
+        &array_size,
+        |b, _| {
+            let udf = ArrayPositions::new();
+            b.iter(|| {
+                black_box(
+                    udf.invoke_with_args(ScalarFunctionArgs {
+                        args: args_not_found.clone(),
+                        arg_fields: arg_fields.clone(),
+                        number_rows: num_rows,
+                        return_field: return_field.clone(),
+                        config_options: config_options.clone(),
+                    })
+                    .unwrap(),
+                )
+            })
+        },
+    );
+
+    group.finish();
+}
+
 fn create_haystack_without_sentinel(
     num_rows: usize,
     array_size: usize,