diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 60715b9c96c0e..a0720cec06313 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -6,7 +6,6 @@ amqps amz amzn anchore -andy apievent apipodspec apk @@ -39,11 +38,8 @@ BADMODE BADSIG BADTIME BADTRUNC -barfoo -barieom basearch baseof -bazqux benefritz bev bfb @@ -61,8 +57,6 @@ bonzai bottlenecked Brandur breadcrumb -bringustogether -bruceg bsd btag btime @@ -76,13 +70,10 @@ bufferpin bvalue bwsnrn bytesize -califrag -califragilistic CAROOT cddl cdylib cef -centiseconds cernan CFFI CGP @@ -91,10 +82,8 @@ chartmuseum chronosphere cicd Claranet -CLF clickhouse clonable -cmds CMK cnf CODEOWNERS @@ -140,7 +129,6 @@ ddsktech ddsource ddtags debuggability -deciseconds decommit dedot deduped @@ -153,7 +141,6 @@ dereferenceable desync developerguide DFm -dhost diffs disintermediate DNSKEY @@ -161,26 +148,20 @@ dnsmsg dogstatsd downcasted dsl -dstaddr dstat -dstport dufs dummyhttp -duser dynamicwireless -dyno ebfcee ector edenhill edns eeyun -efgh Elhage elopment emerg endianess endler -eni enp EOL'ed errorf @@ -193,7 +174,6 @@ eventstreamsink evictingpages evmap ewma -Exabytes examplegroup exitcodes extendedstatus @@ -217,7 +197,6 @@ flamegraph Flatbuffers flatmaps fns -foobarfoobarfoo footgunning Forcepoint freelist @@ -242,6 +221,7 @@ Gregorys groks groupedstats guenter +gzip'd gzip'ed hashring hbb @@ -253,7 +233,6 @@ hostagent hostcall hostpath hoverable -hoverbear httpdeliveryrequestresponse httpevent hugepages @@ -283,7 +262,6 @@ isdst issie iut ixm -jamtur janky JINSPIRED jmxrmi @@ -292,11 +270,9 @@ Jolokia JSONAs jsonify jsonlines -jszwedko kebabcase kernelmode keyclock -keyxxxxx Khal khvzak killall @@ -325,8 +301,6 @@ logseverity logtypes lpop lpush -lucperkins -lukesteensen lvl maj majorly @@ -347,9 +321,7 @@ memsw messagededuplicationid metamethod mezmo -mfoo mgmt -MIDP mingrammer mkto mlua @@ -371,7 +343,6 @@ msv multitenant munging musleabihf -muslueabihf mustprogress mutiple myannotation @@ -392,7 +363,6 @@ nkey nocapture nofree nomac -norc norecurse nosync noundef @@ -402,7 +372,6 @@ npipe nsecs ntoa ntp -nullishness NUMA numbackends NXRR @@ -438,7 +407,6 @@ plds ple podspec Ponge -POSINT preceeded precpu preds @@ -499,6 +467,7 @@ RUSTSEC rustup sandboxing sccache +schema'd schemaless schemars screamingsnakecase @@ -514,14 +483,12 @@ servicebus sev shane shortcode -shost shutsdown SIEM SIGHUPs sinkbuilder sinknetworkbytessent slideover -smartphone Smol smtracking snakecase @@ -536,13 +503,10 @@ spinlock SPOF spog sqlx -srcaddr -srcport SREs sret sse ssekms -ssn stabilises stackdrive stackdriver @@ -555,8 +519,6 @@ subfooter subsecond subtagline summ -Supercalifragilisticexpialidocious -suser sustainability Swich syscalls @@ -568,7 +530,6 @@ Syu tagline tagset Takeaways -targetgroup tarpit tcmalloc telecom @@ -678,5 +639,4 @@ zieme zoog zork zorp -zsherman zulip diff --git a/Cargo.lock b/Cargo.lock index 76e4c912d09ee..dd8d4d9d6ba69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1758,9 +1758,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bigdecimal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", @@ -1932,9 +1932,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.8.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" dependencies = [ "bon-macros", "rustversion", @@ -1942,11 +1942,11 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.8.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.21.3", + "darling", "ident_case", "prettyplease 0.2.37", "proc-macro2 1.0.106", @@ -3062,18 +3062,8 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - -[[package]] -name = "darling" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" -dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core", + "darling_macro", ] [[package]] @@ -3090,38 +3080,13 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling_core" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2 1.0.106", - "quote 1.0.44", - "strsim", - "syn 2.0.117", -] - [[package]] name = "darling_macro" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core 0.20.11", - "quote 1.0.44", - "syn 2.0.117", -] - -[[package]] -name = "darling_macro" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" -dependencies = [ - "darling_core 0.21.3", + "darling_core", "quote 1.0.44", "syn 2.0.117", ] @@ -3307,7 +3272,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ - "darling 0.20.11", + "darling", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -9936,7 +9901,7 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" dependencies = [ - "darling 0.20.11", + "darling", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.117", @@ -12565,7 +12530,7 @@ name = "vector-config-common" version = "0.1.0" dependencies = [ "convert_case 0.8.0", - "darling 0.20.11", + "darling", "proc-macro2 1.0.106", "quote 1.0.44", "serde", @@ -12578,7 +12543,7 @@ dependencies = [ name = "vector-config-macros" version = "0.1.0" dependencies = [ - "darling 0.20.11", + "darling", "proc-macro2 1.0.106", "quote 1.0.44", "serde", diff --git a/changelog.d/24773_support_more_avro_types.fix.md b/changelog.d/24773_support_more_avro_types.fix.md new file mode 100644 index 0000000000000..47d44a6bee32b --- /dev/null +++ b/changelog.d/24773_support_more_avro_types.fix.md @@ -0,0 +1,3 @@ +The Avro codec now supports encoding and decoding Date, Fixed, TimeMillis, TimestampNanos, LocalTimestampNanos values. + +authors: omwbennett diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 90a5fd2780109..9b9f03c1f776a 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -55,6 +55,7 @@ vector-config-macros = { path = "../vector-config-macros", default-features = fa vector-core = { path = "../vector-core", default-features = false, features = ["vrl"] } vector-vrl-functions.workspace = true toml = { version = "0.9.8", optional = true } +uuid.workspace = true [dev-dependencies] futures.workspace = true @@ -65,7 +66,6 @@ similar-asserts = "1.7.0" vector-core = { path = "../vector-core", default-features = false, features = ["vrl", "test"] } rstest = "0.26.1" tracing-test = "0.2.6" -uuid.workspace = true vrl.workspace = true [features] diff --git a/lib/codecs/src/decoding/format/avro.rs b/lib/codecs/src/decoding/format/avro.rs index c21c09de21cfb..25c0227ac4c5c 100644 --- a/lib/codecs/src/decoding/format/avro.rs +++ b/lib/codecs/src/decoding/format/avro.rs @@ -91,11 +91,8 @@ impl From<&AvroDeserializerOptions> for AvroSerializerOptions { pub struct AvroDeserializerOptions { /// The Avro schema definition. /// **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - /// * `Date` /// * `Decimal` /// * `Duration` - /// * `Fixed` - /// * `TimeMillis` #[configurable(metadata( docs::examples = r#"{ "type": "record", "name": "log", "fields": [{ "name": "message", "type": "string" }] }"#, docs::additional_props_description = r#"Supports most avro data types, unsupported data types includes @@ -190,10 +187,8 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { Ok(VrlValue::Array(vector)) } AvroValue::Boolean(boolean) => Ok(VrlValue::from(boolean)), - AvroValue::Bytes(bytes) => Ok(VrlValue::from(bytes)), - AvroValue::Date(_) => Err(vector_common::Error::from( - "AvroValue::Date is not supported", - )), + AvroValue::Bytes(bytes) => Ok(VrlValue::Bytes(Bytes::from(bytes))), + AvroValue::Date(days) => Ok(VrlValue::from(days)), AvroValue::Decimal(_) => Err(vector_common::Error::from( "AvroValue::Decimal is not supported", )), @@ -202,9 +197,7 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { "AvroValue::Duration is not supported", )), AvroValue::Enum(_, string) => Ok(VrlValue::from(string)), - AvroValue::Fixed(_, _) => Err(vector_common::Error::from( - "AvroValue::Fixed is not supported", - )), + AvroValue::Fixed(_, bytes) => Ok(VrlValue::Bytes(Bytes::from(bytes))), AvroValue::Float(float) => Ok(VrlValue::from_f64_or_zero(float as f64)), AvroValue::Int(int) => Ok(VrlValue::from(int)), AvroValue::Long(long) => Ok(VrlValue::from(long)), @@ -221,9 +214,7 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { .map(|v| VrlValue::Object(v.into_iter().collect())), AvroValue::String(string) => Ok(VrlValue::from(string)), AvroValue::TimeMicros(time_micros) => Ok(VrlValue::from(time_micros)), - AvroValue::TimeMillis(_) => Err(vector_common::Error::from( - "AvroValue::TimeMillis is not supported", - )), + AvroValue::TimeMillis(time_millis) => Ok(VrlValue::from(time_millis)), AvroValue::TimestampMicros(ts_micros) => Ok(VrlValue::from(ts_micros)), AvroValue::TimestampMillis(ts_millis) => Ok(VrlValue::from(ts_millis)), AvroValue::Union(_, v) => try_from(*v), @@ -233,12 +224,8 @@ pub fn try_from(value: AvroValue) -> vector_common::Result { AvroValue::BigDecimal(_) => Err(vector_common::Error::from( "AvroValue::BigDecimal is not supported", )), - AvroValue::TimestampNanos(_) => Err(vector_common::Error::from( - "AvroValue::TimestampNanos is not supported", - )), - AvroValue::LocalTimestampNanos(_) => Err(vector_common::Error::from( - "AvroValue::LocalTimestampNanos is not supported", - )), + AvroValue::TimestampNanos(ts_nanos) => Ok(VrlValue::from(ts_nanos)), + AvroValue::LocalTimestampNanos(ts_nanos) => Ok(VrlValue::from(ts_nanos)), } } diff --git a/lib/codecs/src/encoding/format/avro.rs b/lib/codecs/src/encoding/format/avro.rs index 304dcc2e7cc90..1c7daee66f7b8 100644 --- a/lib/codecs/src/encoding/format/avro.rs +++ b/lib/codecs/src/encoding/format/avro.rs @@ -6,6 +6,109 @@ use vector_core::{config::DataType, event::Event, schema}; use crate::encoding::BuildError; +type AvroValue = apache_avro::types::Value; + +/// `apache_avro::to_value` may serialize VRL values into Avro types which later +/// cannot be resolved against certain Avro types +/// (e.g. VRL integer (i64) -> Avro `Long` which cannot be resolved to Avro `Date`). +/// `coerce_logical_types` does a recursive pre-pass to fix such cases. +fn coerce_logical_types( + value: AvroValue, + schema: &apache_avro::Schema, +) -> vector_common::Result { + use apache_avro::Schema; + match (value, schema) { + (AvroValue::Long(days), Schema::Date) => { + i32::try_from(days).map(AvroValue::Date).map_err(|_| { + vector_common::Error::from(format!( + "Avro date value {days} is out of range for i32" + )) + }) + } + (AvroValue::Long(millis), Schema::TimeMillis) => i32::try_from(millis) + .map(AvroValue::TimeMillis) + .map_err(|_| { + vector_common::Error::from(format!( + "Avro time-millis value {millis} is out of range for i32" + )) + }), + (AvroValue::Record(fields), Schema::Record(record_schema)) => { + let fields = fields + .into_iter() + .map(|(name, value)| { + let value = match record_schema.lookup.get(&name) { + Some(index) => { + let field_schema = &record_schema.fields[*index].schema; + coerce_logical_types(value, field_schema)? + } + None => value, + }; + Ok((name, value)) + }) + .collect::>>()?; + Ok(AvroValue::Record(fields)) + } + (AvroValue::Map(entries), Schema::Record(record_schema)) => { + let entries = entries + .into_iter() + .map(|(name, value)| { + let value = match record_schema.lookup.get(&name) { + Some(index) => { + let field_schema = &record_schema.fields[*index].schema; + coerce_logical_types(value, field_schema)? + } + None => value, + }; + Ok((name, value)) + }) + .collect::>()?; + Ok(AvroValue::Map(entries)) + } + (AvroValue::Array(items), Schema::Array(array_schema)) => items + .into_iter() + .map(|item| coerce_logical_types(item, &array_schema.items)) + .collect::, _>>() + .map(AvroValue::Array), + (AvroValue::Map(entries), Schema::Map(map_schema)) => entries + .into_iter() + .map(|(key, value)| { + coerce_logical_types(value, &map_schema.types).map(|value| (key, value)) + }) + .collect::>() + .map(AvroValue::Map), + (AvroValue::Union(index, value), Schema::Union(union_schema)) => { + let schema = union_schema + .variants() + .get(index as usize) + .unwrap_or(schema); + coerce_logical_types(*value, schema) + .map(|value| AvroValue::Union(index, Box::new(value))) + } + (value, Schema::Union(union_schema)) => { + if let Ok(resolved) = value.clone().resolve(schema) { + return Ok(resolved); + } + + let mut last_err = None; + for (index, variant) in union_schema.variants().iter().enumerate() { + match coerce_logical_types(value.clone(), variant) { + Ok(coerced) if coerced.clone().resolve(variant).is_ok() => { + return Ok(AvroValue::Union(index as u32, Box::new(coerced))); + } + Ok(_) => {} + Err(err) => last_err = Some(err), + } + } + + match last_err { + Some(err) => Err(err), + None => Ok(value), + } + } + (value, _) => Ok(value), + } +} + /// Config used to build a `AvroSerializer`. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AvroSerializerConfig { @@ -71,6 +174,7 @@ impl Encoder for AvroSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let log = event.into_log(); let value = apache_avro::to_value(log)?; + let value = coerce_logical_types(value, &self.schema)?; let value = value.resolve(&self.schema)?; let bytes = apache_avro::to_avro_datum(&self.schema, value)?; buffer.put_slice(&bytes); @@ -113,4 +217,158 @@ mod tests { assert_eq!(bytes.freeze(), b"\0\x06bar".as_slice()); } + + #[test] + fn coerce_date_fields_recursively() { + let schema = apache_avro::Schema::parse_str(indoc! {r#" + { + "type": "record", + "name": "Outer", + "fields": [ + { + "name": "direct_date", + "type": {"type": "int", "logicalType": "date"} + }, + { + "name": "inner", + "type": { + "type": "record", + "name": "Inner", + "fields": [ + { + "name": "date", + "type": {"type": "int", "logicalType": "date"} + } + ] + } + }, + { + "name": "record_as_map", + "type": { + "type": "record", + "name": "MapBackedInner", + "fields": [ + { + "name": "date", + "type": {"type": "int", "logicalType": "date"} + } + ] + } + }, + { + "name": "date_array", + "type": { + "type": "array", + "items": {"type": "int", "logicalType": "date"} + } + }, + { + "name": "date_map", + "type": { + "type": "map", + "values": {"type": "int", "logicalType": "date"} + } + }, + { + "name": "union_date", + "type": ["null", {"type": "int", "logicalType": "date"}] + }, + { + "name": "fallback_union_date", + "type": [ + "null", + {"type": "int", "logicalType": "date"}, + "long" + ] + }, + { + "name": "logical_only_union_date", + "type": [ + "null", + {"type": "int", "logicalType": "date"} + ] + } + ] + } + "#}) + .unwrap(); + let value = AvroValue::Record(vec![ + ("direct_date".to_owned(), AvroValue::Long(20_000)), + ( + "inner".to_owned(), + AvroValue::Record(vec![("date".to_owned(), AvroValue::Long(20_001))]), + ), + ( + "record_as_map".to_owned(), + AvroValue::Map( + [("date".to_owned(), AvroValue::Long(20_002))] + .into_iter() + .collect(), + ), + ), + ( + "date_array".to_owned(), + AvroValue::Array(vec![AvroValue::Long(20_003), AvroValue::Long(20_004)]), + ), + ( + "date_map".to_owned(), + AvroValue::Map( + [ + ("first".to_owned(), AvroValue::Long(20_005)), + ("second".to_owned(), AvroValue::Long(20_006)), + ] + .into_iter() + .collect(), + ), + ), + ("union_date".to_owned(), AvroValue::Long(20_007)), + ("fallback_union_date".to_owned(), AvroValue::Long(20_009)), + ( + "logical_only_union_date".to_owned(), + AvroValue::Long(20_008), + ), + ]); + + let value = coerce_logical_types(value, &schema).unwrap(); + let value = value.resolve(&schema).unwrap(); + + assert!(matches!( + value, + AvroValue::Record(fields) if { + matches!(fields[0].1, AvroValue::Date(20_000)) + && matches!( + &fields[1].1, + AvroValue::Record(inner) if matches!(inner[0].1, AvroValue::Date(20_001)) + ) + && matches!( + &fields[2].1, + AvroValue::Record(inner) if matches!(inner[0].1, AvroValue::Date(20_002)) + ) + && matches!( + &fields[3].1, + AvroValue::Array(items) + if matches!(items.as_slice(), [AvroValue::Date(20_003), AvroValue::Date(20_004)]) + ) + && matches!( + &fields[4].1, + AvroValue::Map(entries) + if matches!(entries.get("first"), Some(AvroValue::Date(20_005))) + && matches!(entries.get("second"), Some(AvroValue::Date(20_006))) + ) + && matches!( + &fields[5].1, + AvroValue::Union(1, value) if matches!(value.as_ref(), AvroValue::Date(20_007)) + ) + && matches!( + &fields[6].1, + AvroValue::Union(2, value) + if matches!(value.as_ref(), AvroValue::Long(20_009)) + ) + && matches!( + &fields[7].1, + AvroValue::Union(1, value) if matches!(value.as_ref(), AvroValue::Date(20_008)) + ) + } + )); + } } diff --git a/lib/codecs/tests/avro.rs b/lib/codecs/tests/avro.rs index fff274706d7d9..b56c15dbea61b 100644 --- a/lib/codecs/tests/avro.rs +++ b/lib/codecs/tests/avro.rs @@ -21,9 +21,7 @@ use vector_core::{config::LogNamespace, event::Event}; #[case(true)] #[case(false)] fn roundtrip_avro_fixtures( - #[files("tests/data/avro/generated/*.avro")] - #[exclude(".*(date|fixed|time_millis).avro")] - path: PathBuf, + #[files("tests/data/avro/generated/*.avro")] path: PathBuf, #[case] reserialize: bool, ) { let schema_path = path.as_path().with_extension("avsc"); diff --git a/lib/codecs/tests/bin/generate-avro-fixtures.rs b/lib/codecs/tests/bin/generate-avro-fixtures.rs index 29f22ef0991c0..9dafe1a62cf55 100644 --- a/lib/codecs/tests/bin/generate-avro-fixtures.rs +++ b/lib/codecs/tests/bin/generate-avro-fixtures.rs @@ -142,7 +142,6 @@ fn generate_avro_test_case_string() -> Result<()> { generate_test_case(schema, value, "string") } -#[allow(unused)] fn generate_avro_test_case_fixed() -> Result<()> { let schema = r#" { @@ -280,7 +279,6 @@ fn generate_avro_test_case_record() -> Result<()> { generate_test_case(schema, value, "record") } -#[allow(unused)] fn generate_avro_test_case_date() -> Result<()> { let schema = r#" { @@ -320,7 +318,6 @@ fn generate_avro_test_case_decimal_var() -> Result<()> { generate_test_case_from_value(schema, record, "decimal_var") } -#[allow(unused)] fn generate_avro_test_case_time_millis() -> Result<()> { let schema = r#" { @@ -441,6 +438,46 @@ fn generate_avro_test_case_local_timestamp_micros() -> Result<()> { generate_test_case(schema, value, "local-timestamp_micros") } +fn generate_avro_test_case_timestamp_nanos() -> Result<()> { + let schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "timestamp_nanos_field", "type": "long", "logicalType": "timestamp-nanos"} + ] + } + "#; + #[derive(Debug, Serialize, Deserialize, Clone)] + struct Test { + timestamp_nanos_field: i64, + } + let value = Test { + timestamp_nanos_field: 1697445291056567890i64, + }; + generate_test_case(schema, value, "timestamp_nanos") +} + +fn generate_avro_test_case_local_timestamp_nanos() -> Result<()> { + let schema = r#" + { + "type": "record", + "name": "test", + "fields": [ + {"name": "local_timestamp_nanos_field", "type": "long", "logicalType": "local-timestamp-nanos"} + ] + } + "#; + #[derive(Debug, Serialize, Deserialize, Clone)] + struct Test { + local_timestamp_nanos_field: i64, + } + let value = Test { + local_timestamp_nanos_field: 1697445291056567890i64, + }; + generate_test_case(schema, value, "local-timestamp_nanos") +} + fn generate_avro_test_case_uuid() -> Result<()> { let schema = r#" { @@ -476,7 +513,7 @@ fn generate_test_case_from_value(schema: &str, value: Value, filename: &str) -> let mut schema_file = File::create(format!("{FIXTURES_PATH}/{filename}.avsc"))?; let mut avro_file = File::create(format!("{FIXTURES_PATH}/{filename}.avro"))?; - schema_file.write_all(schema.canonical_form().as_bytes())?; + schema_file.write_all(serde_json::to_string(&schema)?.as_bytes())?; avro_file.write_all(&bytes)?; Ok(()) } @@ -488,8 +525,10 @@ fn main() -> Result<()> { generate_avro_test_case_array()?; generate_avro_test_case_boolean()?; generate_avro_test_case_bytes()?; + generate_avro_test_case_date()?; generate_avro_test_case_double()?; generate_avro_test_case_enum()?; + generate_avro_test_case_fixed()?; generate_avro_test_case_float()?; generate_avro_test_case_int()?; generate_avro_test_case_long()?; @@ -497,10 +536,13 @@ fn main() -> Result<()> { generate_avro_test_case_record()?; generate_avro_test_case_string()?; generate_avro_test_case_time_micros()?; + generate_avro_test_case_time_millis()?; generate_avro_test_case_timestamp_micros()?; generate_avro_test_case_timestamp_millis()?; + generate_avro_test_case_timestamp_nanos()?; generate_avro_test_case_local_timestamp_micros()?; generate_avro_test_case_local_timestamp_millis()?; + generate_avro_test_case_local_timestamp_nanos()?; generate_avro_test_case_union()?; generate_avro_test_case_uuid()?; Ok(()) diff --git a/lib/codecs/tests/data/avro/generated/array.avsc b/lib/codecs/tests/data/avro/generated/array.avsc index 41f098e5deafe..3ce68e7b38060 100644 --- a/lib/codecs/tests/data/avro/generated/array.avsc +++ b/lib/codecs/tests/data/avro/generated/array.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"array_field","type":{"type":"array","items":"string"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"array_field","type":{"type":"array","items":"string"},"items":"string"}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/boolean.avsc b/lib/codecs/tests/data/avro/generated/boolean.avsc index aa04929ca9db4..d0a8839a6a2f1 100644 --- a/lib/codecs/tests/data/avro/generated/boolean.avsc +++ b/lib/codecs/tests/data/avro/generated/boolean.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"bool_field","type":"boolean"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"bool_field","type":"boolean","default":false}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/bytes.avsc b/lib/codecs/tests/data/avro/generated/bytes.avsc index 2d8d0acefc72a..c34b6ff57321c 100644 --- a/lib/codecs/tests/data/avro/generated/bytes.avsc +++ b/lib/codecs/tests/data/avro/generated/bytes.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"bytes_field","type":"bytes"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"bytes_field","type":"bytes"}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/date.avro b/lib/codecs/tests/data/avro/generated/date.avro new file mode 100644 index 0000000000000..b067210aeac93 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/date.avro @@ -0,0 +1 @@ +ü² \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/date.avsc b/lib/codecs/tests/data/avro/generated/date.avsc new file mode 100644 index 0000000000000..c7a70afe9be1a --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/date.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"date_field","type":{"type":"int","logicalType":"date"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/double.avsc b/lib/codecs/tests/data/avro/generated/double.avsc index 60dfa5b2aab17..7efdd059d60b8 100644 --- a/lib/codecs/tests/data/avro/generated/double.avsc +++ b/lib/codecs/tests/data/avro/generated/double.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"double_field","type":"double"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"double_field","type":"double","default":0}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/enum.avsc b/lib/codecs/tests/data/avro/generated/enum.avsc index 5d7468754da98..59d5d0e34dad2 100644 --- a/lib/codecs/tests/data/avro/generated/enum.avsc +++ b/lib/codecs/tests/data/avro/generated/enum.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"enum_field","type":{"name":"enum_field","type":"enum","symbols":["Spades","Hearts","Diamonds","Clubs"]}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"enum_field","type":{"type":"enum","name":"enum_field","symbols":["Spades","Hearts","Diamonds","Clubs"]}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/fixed.avro b/lib/codecs/tests/data/avro/generated/fixed.avro new file mode 100644 index 0000000000000..50fd96c66abba --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/fixed.avro @@ -0,0 +1 @@ +1019181716151413 \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/fixed.avsc b/lib/codecs/tests/data/avro/generated/fixed.avsc new file mode 100644 index 0000000000000..ee03066a96081 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/fixed.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"fixed_field","type":{"type":"fixed","name":"fixed_field","size":16}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/float.avsc b/lib/codecs/tests/data/avro/generated/float.avsc index 9a7f836ddfe0d..aa0d287e4dfc3 100644 --- a/lib/codecs/tests/data/avro/generated/float.avsc +++ b/lib/codecs/tests/data/avro/generated/float.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"float_field","type":"float"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"float_field","type":"float","default":0}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/int.avsc b/lib/codecs/tests/data/avro/generated/int.avsc index af009f13394f2..134feda0ea323 100644 --- a/lib/codecs/tests/data/avro/generated/int.avsc +++ b/lib/codecs/tests/data/avro/generated/int.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"int_field","type":"int"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"int_field","type":"int","default":0}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/local-timestamp_micros.avsc b/lib/codecs/tests/data/avro/generated/local-timestamp_micros.avsc index 35d5441f13078..01d0dcf5eca96 100644 --- a/lib/codecs/tests/data/avro/generated/local-timestamp_micros.avsc +++ b/lib/codecs/tests/data/avro/generated/local-timestamp_micros.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"local_timestamp_micros_field","type":{"type":"long","logicalType":"local-timestamp-micros"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"local_timestamp_micros_field","type":{"type":"long","logicalType":"local-timestamp-micros"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/local-timestamp_millis.avsc b/lib/codecs/tests/data/avro/generated/local-timestamp_millis.avsc index 51ec12c1e477e..39722937a5d13 100644 --- a/lib/codecs/tests/data/avro/generated/local-timestamp_millis.avsc +++ b/lib/codecs/tests/data/avro/generated/local-timestamp_millis.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"local_timestamp_millis_field","type":{"type":"long","logicalType":"local-timestamp-millis"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"local_timestamp_millis_field","type":{"type":"long","logicalType":"local-timestamp-millis"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/local-timestamp_nanos.avro b/lib/codecs/tests/data/avro/generated/local-timestamp_nanos.avro new file mode 100644 index 0000000000000..bb475c1f66117 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/local-timestamp_nanos.avro @@ -0,0 +1 @@ +¤Ù°ñõßÄŽ/ \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/local-timestamp_nanos.avsc b/lib/codecs/tests/data/avro/generated/local-timestamp_nanos.avsc new file mode 100644 index 0000000000000..72fc1dc5ea873 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/local-timestamp_nanos.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"local_timestamp_nanos_field","type":{"type":"long","logicalType":"local-timestamp-nanos"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/long.avsc b/lib/codecs/tests/data/avro/generated/long.avsc index e4a052e4eacef..4976e3d40a8fb 100644 --- a/lib/codecs/tests/data/avro/generated/long.avsc +++ b/lib/codecs/tests/data/avro/generated/long.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"long_field","type":"long"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"long_field","type":"long","default":0}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/map.avsc b/lib/codecs/tests/data/avro/generated/map.avsc index cc3e2ce9003dd..69441d98fa538 100644 --- a/lib/codecs/tests/data/avro/generated/map.avsc +++ b/lib/codecs/tests/data/avro/generated/map.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"map_field","type":{"type":"map","values":"long"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"map_field","type":{"type":"map","values":"long","default":{}},"default":{},"values":"long"}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/record.avsc b/lib/codecs/tests/data/avro/generated/record.avsc index 04632d003a183..e644a5f586fe9 100644 --- a/lib/codecs/tests/data/avro/generated/record.avsc +++ b/lib/codecs/tests/data/avro/generated/record.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"name","type":"string"},{"name":"age","type":"int"}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/string.avsc b/lib/codecs/tests/data/avro/generated/string.avsc index b6efaad1e7e11..d26a0c1f81e07 100644 --- a/lib/codecs/tests/data/avro/generated/string.avsc +++ b/lib/codecs/tests/data/avro/generated/string.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"string_field","type":"string"}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"string_field","type":"string"}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/time_micros.avsc b/lib/codecs/tests/data/avro/generated/time_micros.avsc index 6624415706d0e..455d3e9cd2ada 100644 --- a/lib/codecs/tests/data/avro/generated/time_micros.avsc +++ b/lib/codecs/tests/data/avro/generated/time_micros.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"time_micros_field","type":{"type":"long","logicalType":"time-micros"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"time_micros_field","type":{"type":"long","logicalType":"time-micros"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/time_millis.avro b/lib/codecs/tests/data/avro/generated/time_millis.avro new file mode 100644 index 0000000000000..c1aee88444369 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/time_millis.avro @@ -0,0 +1 @@ +¶¡†9 \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/time_millis.avsc b/lib/codecs/tests/data/avro/generated/time_millis.avsc new file mode 100644 index 0000000000000..9a281db2c6e99 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/time_millis.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"time_millis_field","type":{"type":"int","logicalType":"time-millis"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/timestamp_micros.avsc b/lib/codecs/tests/data/avro/generated/timestamp_micros.avsc index b466968145c9f..d51fdda84588b 100644 --- a/lib/codecs/tests/data/avro/generated/timestamp_micros.avsc +++ b/lib/codecs/tests/data/avro/generated/timestamp_micros.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"timestamp_micros_field","type":{"type":"long","logicalType":"timestamp-micros"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"timestamp_micros_field","type":{"type":"long","logicalType":"timestamp-micros"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/timestamp_millis.avsc b/lib/codecs/tests/data/avro/generated/timestamp_millis.avsc index eea96da274baf..5a42c53182c7d 100644 --- a/lib/codecs/tests/data/avro/generated/timestamp_millis.avsc +++ b/lib/codecs/tests/data/avro/generated/timestamp_millis.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"timestamp_millis_field","type":{"type":"long","logicalType":"timestamp-millis"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"timestamp_millis_field","type":{"type":"long","logicalType":"timestamp-millis"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/timestamp_nanos.avro b/lib/codecs/tests/data/avro/generated/timestamp_nanos.avro new file mode 100644 index 0000000000000..bb475c1f66117 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/timestamp_nanos.avro @@ -0,0 +1 @@ +¤Ù°ñõßÄŽ/ \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/timestamp_nanos.avsc b/lib/codecs/tests/data/avro/generated/timestamp_nanos.avsc new file mode 100644 index 0000000000000..e03634505d251 --- /dev/null +++ b/lib/codecs/tests/data/avro/generated/timestamp_nanos.avsc @@ -0,0 +1 @@ +{"type":"record","name":"test","fields":[{"name":"timestamp_nanos_field","type":{"type":"long","logicalType":"timestamp-nanos"}}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/union.avsc b/lib/codecs/tests/data/avro/generated/union.avsc index b67afd5112a6b..bd31d552c42f2 100644 --- a/lib/codecs/tests/data/avro/generated/union.avsc +++ b/lib/codecs/tests/data/avro/generated/union.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"union_field","type":["string","int"]}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"union_field","type":["string","int"]}]} \ No newline at end of file diff --git a/lib/codecs/tests/data/avro/generated/uuid.avsc b/lib/codecs/tests/data/avro/generated/uuid.avsc index ca1b0bf400de1..d20f63c4d503a 100644 --- a/lib/codecs/tests/data/avro/generated/uuid.avsc +++ b/lib/codecs/tests/data/avro/generated/uuid.avsc @@ -1 +1 @@ -{"name":"test","type":"record","fields":[{"name":"uuid_field","type":{"type":"string","logicalType":"uuid"}}]} \ No newline at end of file +{"type":"record","name":"test","fields":[{"name":"uuid_field","type":{"type":"string","logicalType":"uuid"}}]} \ No newline at end of file diff --git a/website/cue/reference/components/sinks/generated/greptimedb_logs.cue b/website/cue/reference/components/sinks/generated/greptimedb_logs.cue index ae7e4a444700c..f93188f9d6a3e 100644 --- a/website/cue/reference/components/sinks/generated/greptimedb_logs.cue +++ b/website/cue/reference/components/sinks/generated/greptimedb_logs.cue @@ -153,7 +153,8 @@ generated: components: sinks: greptimedb_logs: configuration: { """ required: false type: object: { - examples: [{}] + examples: [{}, + ] options: "*": { description: "Extra header key-value pairs." required: true diff --git a/website/cue/reference/components/sinks/generated/websocket_server.cue b/website/cue/reference/components/sinks/generated/websocket_server.cue index 28e5823c8c7e5..fead30ffb7cc4 100644 --- a/website/cue/reference/components/sinks/generated/websocket_server.cue +++ b/website/cue/reference/components/sinks/generated/websocket_server.cue @@ -594,11 +594,8 @@ generated: components: sinks: websocket_server: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/amqp.cue b/website/cue/reference/components/sources/generated/amqp.cue index ef71f410f0e29..0822b13104717 100644 --- a/website/cue/reference/components/sources/generated/amqp.cue +++ b/website/cue/reference/components/sources/generated/amqp.cue @@ -62,11 +62,8 @@ generated: components: sources: amqp: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue b/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue index 4464a2305412e..ba43f3f6f3656 100644 --- a/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sources/generated/aws_kinesis_firehose.cue @@ -65,11 +65,8 @@ generated: components: sources: aws_kinesis_firehose: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/aws_s3.cue b/website/cue/reference/components/sources/generated/aws_s3.cue index 95730d6c68ca6..c423876870fa7 100644 --- a/website/cue/reference/components/sources/generated/aws_s3.cue +++ b/website/cue/reference/components/sources/generated/aws_s3.cue @@ -180,11 +180,8 @@ generated: components: sources: aws_s3: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/aws_sqs.cue b/website/cue/reference/components/sources/generated/aws_sqs.cue index b52cfcea28a31..ce4c63a9dfaa9 100644 --- a/website/cue/reference/components/sources/generated/aws_sqs.cue +++ b/website/cue/reference/components/sources/generated/aws_sqs.cue @@ -175,11 +175,8 @@ generated: components: sources: aws_sqs: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/datadog_agent.cue b/website/cue/reference/components/sources/generated/datadog_agent.cue index 70bd89821cb1f..5a8c17ca73517 100644 --- a/website/cue/reference/components/sources/generated/datadog_agent.cue +++ b/website/cue/reference/components/sources/generated/datadog_agent.cue @@ -47,11 +47,8 @@ generated: components: sources: datadog_agent: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/demo_logs.cue b/website/cue/reference/components/sources/generated/demo_logs.cue index e1227f9fc26ca..fe63390527a76 100644 --- a/website/cue/reference/components/sources/generated/demo_logs.cue +++ b/website/cue/reference/components/sources/generated/demo_logs.cue @@ -26,11 +26,8 @@ generated: components: sources: demo_logs: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/exec.cue b/website/cue/reference/components/sources/generated/exec.cue index a5380b6c01f1d..07a7b1fce82d3 100644 --- a/website/cue/reference/components/sources/generated/exec.cue +++ b/website/cue/reference/components/sources/generated/exec.cue @@ -27,11 +27,8 @@ generated: components: sources: exec: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/file_descriptor.cue b/website/cue/reference/components/sources/generated/file_descriptor.cue index 84ecfd0d85e94..4d9566f56aa07 100644 --- a/website/cue/reference/components/sources/generated/file_descriptor.cue +++ b/website/cue/reference/components/sources/generated/file_descriptor.cue @@ -17,11 +17,8 @@ generated: components: sources: file_descriptor: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/gcp_pubsub.cue b/website/cue/reference/components/sources/generated/gcp_pubsub.cue index 74ceb80df85ff..6f7db9c8aa724 100644 --- a/website/cue/reference/components/sources/generated/gcp_pubsub.cue +++ b/website/cue/reference/components/sources/generated/gcp_pubsub.cue @@ -93,11 +93,8 @@ generated: components: sources: gcp_pubsub: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/heroku_logs.cue b/website/cue/reference/components/sources/generated/heroku_logs.cue index cb44587b15e4b..dfb83e028e8f0 100644 --- a/website/cue/reference/components/sources/generated/heroku_logs.cue +++ b/website/cue/reference/components/sources/generated/heroku_logs.cue @@ -90,11 +90,8 @@ generated: components: sources: heroku_logs: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/http.cue b/website/cue/reference/components/sources/generated/http.cue index 259c800829438..7620a030d5cf9 100644 --- a/website/cue/reference/components/sources/generated/http.cue +++ b/website/cue/reference/components/sources/generated/http.cue @@ -94,11 +94,8 @@ generated: components: sources: http: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/http_client.cue b/website/cue/reference/components/sources/generated/http_client.cue index 6631650e47fb5..a1ac12da0fb5d 100644 --- a/website/cue/reference/components/sources/generated/http_client.cue +++ b/website/cue/reference/components/sources/generated/http_client.cue @@ -228,11 +228,8 @@ generated: components: sources: http_client: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/http_server.cue b/website/cue/reference/components/sources/generated/http_server.cue index c84425dd8db2c..9a29a0aa0b2cf 100644 --- a/website/cue/reference/components/sources/generated/http_server.cue +++ b/website/cue/reference/components/sources/generated/http_server.cue @@ -94,11 +94,8 @@ generated: components: sources: http_server: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/kafka.cue b/website/cue/reference/components/sources/generated/kafka.cue index 7a0ebfdcf1006..a6715ac5216f4 100644 --- a/website/cue/reference/components/sources/generated/kafka.cue +++ b/website/cue/reference/components/sources/generated/kafka.cue @@ -71,11 +71,8 @@ generated: components: sources: kafka: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/mqtt.cue b/website/cue/reference/components/sources/generated/mqtt.cue index 0bcd847fd4717..852a8aea89c51 100644 --- a/website/cue/reference/components/sources/generated/mqtt.cue +++ b/website/cue/reference/components/sources/generated/mqtt.cue @@ -22,11 +22,8 @@ generated: components: sources: mqtt: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/nats.cue b/website/cue/reference/components/sources/generated/nats.cue index c57371bafaa94..22c0ca0bf676a 100644 --- a/website/cue/reference/components/sources/generated/nats.cue +++ b/website/cue/reference/components/sources/generated/nats.cue @@ -114,11 +114,8 @@ generated: components: sources: nats: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/pulsar.cue b/website/cue/reference/components/sources/generated/pulsar.cue index f51e1a449dbcb..114c2b2165634 100644 --- a/website/cue/reference/components/sources/generated/pulsar.cue +++ b/website/cue/reference/components/sources/generated/pulsar.cue @@ -120,11 +120,8 @@ generated: components: sources: pulsar: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/redis.cue b/website/cue/reference/components/sources/generated/redis.cue index 4ddff3bf52d73..f1ac047ae1631 100644 --- a/website/cue/reference/components/sources/generated/redis.cue +++ b/website/cue/reference/components/sources/generated/redis.cue @@ -32,11 +32,8 @@ generated: components: sources: redis: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/socket.cue b/website/cue/reference/components/sources/generated/socket.cue index 63495e4a1fbaf..d99cf52681146 100644 --- a/website/cue/reference/components/sources/generated/socket.cue +++ b/website/cue/reference/components/sources/generated/socket.cue @@ -34,11 +34,8 @@ generated: components: sources: socket: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/stdin.cue b/website/cue/reference/components/sources/generated/stdin.cue index 72de586122b44..f3f131e9e50f7 100644 --- a/website/cue/reference/components/sources/generated/stdin.cue +++ b/website/cue/reference/components/sources/generated/stdin.cue @@ -17,11 +17,8 @@ generated: components: sources: stdin: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"] diff --git a/website/cue/reference/components/sources/generated/websocket.cue b/website/cue/reference/components/sources/generated/websocket.cue index c770d5558e71a..16bd04415ddee 100644 --- a/website/cue/reference/components/sources/generated/websocket.cue +++ b/website/cue/reference/components/sources/generated/websocket.cue @@ -204,11 +204,8 @@ generated: components: sources: websocket: configuration: { description: """ The Avro schema definition. **Note**: The following [`apache_avro::types::Value`] variants are *not* supported: - * `Date` * `Decimal` * `Duration` - * `Fixed` - * `TimeMillis` """ required: true type: string: examples: ["{ \"type\": \"record\", \"name\": \"log\", \"fields\": [{ \"name\": \"message\", \"type\": \"string\" }] }"]