feat: Generate transform in WriteContext w.r.t column mapping (delta-io#1862)

## 🥞 Stacked PR
Use this
[link](https://github.com/delta-io/delta-kernel-rs/pull/1862/files) to
review incremental changes.
-
[**stack/correct-transform**](delta-io#1862)
[[Files
changed](https://github.com/delta-io/delta-kernel-rs/pull/1862/files)]
-
[stack/cm-partition](delta-io#1870)
[[Files
changed](https://github.com/delta-io/delta-kernel-rs/pull/1870/files/6b36ecc5022a5be295d128fe6b66192b74d39614..76691de21b81671453a8e52a1fd58b58fdcbf1fe)]
-
[stack/support-cm-write](delta-io#1863)
[[Files
changed](https://github.com/delta-io/delta-kernel-rs/pull/1863/files/76691de21b81671453a8e52a1fd58b58fdcbf1fe..1e938eb9de2f5f8a52943ae794984585867f31a5)]
-
[stack/support-CM-with-flag](delta-io#1910)
[[Files
changed](https://github.com/delta-io/delta-kernel-rs/pull/1910/files/6b36ecc5022a5be295d128fe6b66192b74d39614..7e184b835db7ac1142cb878524561847007dc297)]

---------
## What changes are proposed in this pull request?
Add with_dropped_field_if_exists to Transform for optionally dropping
fields that may not exist in the input, and use it in
generate_logical_to_physical to drop partition columns without erroring
when the input data doesn't contain them.

Change `WriteContext::logical_to_physical` will now correctly rename
nested struct fields to their physical names under column mapping.


<!--
**Uncomment** this section if there are any changes affecting public
APIs. Else, **delete** this section.
### This PR affects the following public APIs
If there are breaking changes, please ensure the `breaking-changes`
label gets added by CI, and describe why the changes are needed.
Note that _new_ public APIs are not considered breaking.
-->

## How was this change tested?
Added test to verify the renaming successful and
`with_dropped_field_if_exists` works as expected

Author: dengsh12

kernel/src/engine/arrow_expression/evaluate_expression.rs

@@ -199,10 +199,15 @@ fn evaluate_transform_expression(
         }
     }
 
-    // Verify that all field transforms were used
-    if used_field_transforms != transform.field_transforms.len() {
+    // Verify that all non-optional field transforms were used
+    let required_count = transform
+        .field_transforms
+        .values()
+        .filter(|ft| !ft.optional)
+        .count();
+    if used_field_transforms < required_count {
         return Err(Error::generic(
-            "Some field transforms reference invalid input field names",
+            "Some non-optional field transforms reference invalid input field names",
         ));
     }
 
@@ -987,6 +992,69 @@ mod tests {
             .contains("Data type is required"));
     }
 
+    #[test]
+    fn test_drop_field_if_exists_present() {
+        let batch = create_test_batch();
+        let transform = Transform::new_top_level().with_dropped_field_if_exists("a");
+        let output_schema = StructType::new_unchecked(vec![
+            StructField::not_null("b", DataType::INTEGER),
+            StructField::not_null("c", DataType::INTEGER),
+        ]);
+        let expr = Expr::Transform(transform);
+        let result = evaluate_expression(
+            &expr,
+            &batch,
+            Some(&DataType::Struct(Box::new(output_schema))),
+        )
+        .unwrap();
+        let result = result.as_any().downcast_ref::<StructArray>().unwrap();
+        validate_i32_column(result, 0, &[10, 20, 30]);
+        validate_i32_column(result, 1, &[100, 200, 300]);
+    }
+
+    #[test]
+    fn test_drop_field_if_exists_missing() {
+        let batch = create_test_batch();
+        let transform = Transform::new_top_level().with_dropped_field_if_exists("nonexistent");
+        let output_schema = StructType::new_unchecked(vec![
+            StructField::not_null("a", DataType::INTEGER),
+            StructField::not_null("b", DataType::INTEGER),
+            StructField::not_null("c", DataType::INTEGER),
+        ]);
+        let expr = Expr::Transform(transform);
+        let result = evaluate_expression(
+            &expr,
+            &batch,
+            Some(&DataType::Struct(Box::new(output_schema))),
+        )
+        .unwrap();
+        let result = result.as_any().downcast_ref::<StructArray>().unwrap();
+        validate_i32_column(result, 0, &[1, 2, 3]);
+        validate_i32_column(result, 1, &[10, 20, 30]);
+        validate_i32_column(result, 2, &[100, 200, 300]);
+    }
+
+    #[test]
+    fn test_drop_field_non_optional_missing_still_errors() {
+        let batch = create_test_batch();
+        let transform = Transform::new_top_level().with_dropped_field("nonexistent");
+        let output_schema = StructType::new_unchecked(vec![
+            StructField::not_null("a", DataType::INTEGER),
+            StructField::not_null("b", DataType::INTEGER),
+            StructField::not_null("c", DataType::INTEGER),
+        ]);
+        let expr = Expr::Transform(transform);
+        let result = evaluate_expression(
+            &expr,
+            &batch,
+            Some(&DataType::Struct(Box::new(output_schema))),
+        );
+        assert!(result
+            .unwrap_err()
+            .to_string()
+            .contains("reference invalid input field names"));
+    }
+
     #[test]
     fn test_struct_expression_schema_validation() {
         let batch = create_test_batch();

kernel/src/expressions/mod.rs

@@ -336,6 +336,9 @@ pub struct FieldTransform {
     pub exprs: Vec<ExpressionRef>,
     /// If true, the output expressions replace the input field instead of following after it.
     pub is_replace: bool,
+    /// If true, this transform is silently ignored when the target field does not exist in the
+    /// input. Otherwise, a missing target field produces an error.
+    pub optional: bool,
 }
 
 /// A transformation that efficiently represents sparse modifications to struct schemas.
@@ -382,6 +385,14 @@ impl Transform {
         self
     }
 
+    /// Like [`Self::with_dropped_field`], but silently ignored if the field does not exist.
+    pub fn with_dropped_field_if_exists(mut self, name: impl Into<String>) -> Self {
+        let field_transform = self.field_transform(name);
+        field_transform.is_replace = true;
+        field_transform.optional = true;
+        self
+    }
+
     /// Specifies an expression to replace a field with.
     pub fn with_replaced_field(mut self, name: impl Into<String>, expr: ExpressionRef) -> Self {
         let field_transform = self.field_transform(name);