chore!: add missing reader and writer features, make feature enums internal_api (delta-io#998)

## What changes are proposed in this pull request?

This PR adds the missing features `clustering` and `variantType` to the
reader and writer feature enums.

delta-rs also needs to be able to inspect reader / writer features since
the feature support is still somewhat decoupled from kernel. To
facilitate adoption of kernel we therefore also expose `reader_features`
and `writer_features` on the `Protocol` action behind the `internal-api`
flag.

### This PR affects the following public APIs

exposes `reader_features` and `writer_features` on the `Protocol` action
behind the `internal-api` flag. reader/writer feature enums are now
`internal-api` only (not `pub`)

## How was this change tested?

new roundtrip cases for new variants.

---------

Co-authored-by: Zach Schuermann <zachary.zvs@gmail.com>

Author: roeap

kernel/src/actions/mod.rs

@@ -298,11 +298,13 @@ impl Protocol {
     }
 
     /// Get the reader features for the protocol
+    #[internal_api]
     pub(crate) fn reader_features(&self) -> Option<&[ReaderFeature]> {
         self.reader_features.as_deref()
     }
 
     /// Get the writer features for the protocol
+    #[internal_api]
     pub(crate) fn writer_features(&self) -> Option<&[WriterFeature]> {
         self.writer_features.as_deref()
     }

kernel/src/table_features/mod.rs

@@ -5,6 +5,7 @@ use strum::{AsRefStr, Display as StrumDisplay, EnumCount, EnumString};
 
 use crate::schema::derive_macro_utils::ToDataType;
 use crate::schema::DataType;
+use delta_kernel_derive::internal_api;
 
 pub(crate) use column_mapping::column_mapping_mode;
 pub use column_mapping::{validate_schema_column_mapping, ColumnMappingMode};
@@ -33,7 +34,8 @@ mod timestamp_ntz;
 )]
 #[strum(serialize_all = "camelCase")]
 #[serde(rename_all = "camelCase")]
-pub enum ReaderFeature {
+#[internal_api]
+pub(crate) enum ReaderFeature {
     /// Mapping of one column to another
     ColumnMapping,
     /// Deletion vectors for merge, update, delete
@@ -52,6 +54,8 @@ pub enum ReaderFeature {
     /// vacuumProtocolCheck ReaderWriter feature ensures consistent application of reader and writer
     /// protocol checks during VACUUM operations
     VacuumProtocolCheck,
+    /// This feature enables support for the variant data type, which stores semi-structured data.
+    VariantType,
     #[serde(untagged)]
     #[strum(default)]
     Unknown(String),
@@ -77,7 +81,8 @@ pub enum ReaderFeature {
 )]
 #[strum(serialize_all = "camelCase")]
 #[serde(rename_all = "camelCase")]
-pub enum WriterFeature {
+#[internal_api]
+pub(crate) enum WriterFeature {
     /// Append Only Tables
     AppendOnly,
     /// Table invariants
@@ -118,6 +123,13 @@ pub enum WriterFeature {
     /// vacuumProtocolCheck ReaderWriter feature ensures consistent application of reader and writer
     /// protocol checks during VACUUM operations
     VacuumProtocolCheck,
+    /// The Clustered Table feature facilitates the physical clustering of rows
+    /// that share similar values on a predefined set of clustering columns.
+    #[strum(serialize = "clustering")]
+    #[serde(rename = "clustering")]
+    ClusteredTable,
+    /// This feature enables support for the variant data type, which stores semi-structured data.
+    VariantType,
     #[serde(untagged)]
     #[strum(default)]
     Unknown(String),
@@ -221,6 +233,7 @@ mod tests {
             (ReaderFeature::TypeWideningPreview, "typeWidening-preview"),
             (ReaderFeature::V2Checkpoint, "v2Checkpoint"),
             (ReaderFeature::VacuumProtocolCheck, "vacuumProtocolCheck"),
+            (ReaderFeature::VariantType, "variantType"),
             (ReaderFeature::unknown("something"), "something"),
         ];
 
@@ -260,6 +273,8 @@ mod tests {
             (WriterFeature::IcebergCompatV1, "icebergCompatV1"),
             (WriterFeature::IcebergCompatV2, "icebergCompatV2"),
             (WriterFeature::VacuumProtocolCheck, "vacuumProtocolCheck"),
+            (WriterFeature::ClusteredTable, "clustering"),
+            (WriterFeature::VariantType, "variantType"),
             (WriterFeature::unknown("something"), "something"),
         ];