Skip to content

Commit 050320d

Browse files
authored
chore: Update spark_expressions_support.md doc (#4165)
1 parent dd05146 commit 050320d

1 file changed

Lines changed: 131 additions & 17 deletions

File tree

docs/source/contributor-guide/spark_expressions_support.md

Lines changed: 131 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,25 @@
1919

2020
# Supported Spark Expressions
2121

22+
## How to Read This Document
23+
24+
- A function marked with `[x]` has a native implementation in Comet and does not fall back to Spark by default.
25+
- A function marked with `[ ]` has no native Comet implementation and falls back to Spark.
26+
27+
> **Note:** Some functions listed as supported may still be incompatible with Spark in
28+
> certain cases (data types, modes, edge values) and fall back to Spark at runtime. Full
29+
> per-function details are documented in the
30+
> [Compatibility Guide](https://datafusion.apache.org/comet/user-guide/latest/compatibility/expressions/index.html).
31+
2232
### agg_funcs
2333

2434
- [x] any
2535
- [x] any_value
2636
- [ ] approx_count_distinct
2737
- [ ] approx_percentile
38+
- [ ] approx_top_k
39+
- [ ] approx_top_k_accumulate
40+
- [ ] approx_top_k_combine
2841
- [ ] array_agg
2942
- [x] avg
3043
- [x] bit_and
@@ -46,9 +59,30 @@
4659
- [ ] grouping
4760
- [ ] grouping_id
4861
- [ ] histogram_numeric
62+
- [ ] hll_sketch_agg
63+
- [ ] hll_union_agg
64+
- [ ] kll_sketch_agg_bigint
65+
- [ ] kll_sketch_agg_double
66+
- [ ] kll_sketch_agg_float
67+
- [ ] kll_sketch_get_n_bigint
68+
- [ ] kll_sketch_get_n_double
69+
- [ ] kll_sketch_get_n_float
70+
- [ ] kll_sketch_get_quantile_bigint
71+
- [ ] kll_sketch_get_quantile_double
72+
- [ ] kll_sketch_get_quantile_float
73+
- [ ] kll_sketch_get_rank_bigint
74+
- [ ] kll_sketch_get_rank_double
75+
- [ ] kll_sketch_get_rank_float
76+
- [ ] kll_sketch_merge_bigint
77+
- [ ] kll_sketch_merge_double
78+
- [ ] kll_sketch_merge_float
79+
- [ ] kll_sketch_to_string_bigint
80+
- [ ] kll_sketch_to_string_double
81+
- [ ] kll_sketch_to_string_float
4982
- [ ] kurtosis
5083
- [x] last
5184
- [x] last_value
85+
- [ ] listagg
5286
- [x] max
5387
- [ ] max_by
5488
- [x] mean
@@ -58,6 +92,8 @@
5892
- [ ] mode
5993
- [ ] percentile
6094
- [ ] percentile_approx
95+
- [ ] percentile_cont
96+
- [ ] percentile_disc
6197
- [x] regr_avgx
6298
- [x] regr_avgy
6399
- [x] regr_count
@@ -73,7 +109,11 @@
73109
- [x] stddev
74110
- [x] stddev_pop
75111
- [x] stddev_samp
112+
- [ ] string_agg
76113
- [x] sum
114+
- [ ] theta_intersection_agg
115+
- [ ] theta_sketch_agg
116+
- [ ] theta_union_agg
77117
- [ ] try_avg
78118
- [ ] try_sum
79119
- [x] var_pop
@@ -100,6 +140,7 @@
100140
- [x] array_max
101141
- [ ] array_min
102142
- [x] array_position
143+
- [ ] array_prepend
103144
- [x] array_remove
104145
- [x] array_repeat
105146
- [x] array_union
@@ -115,15 +156,18 @@
115156

116157
### bitwise_funcs
117158

118-
- [x] &
119-
- [x] ^
159+
- [x] `&`
160+
- [ ] `<<`
161+
- [ ] `>>`
162+
- [ ] `>>>`
163+
- [x] `^`
120164
- [ ] bit_count
121165
- [ ] bit_get
122166
- [ ] getbit
123167
- [x] shiftright
124168
- [ ] shiftrightunsigned
125-
- [x] |
126-
- [x] ~
169+
- [x] `|`
170+
- [x] `~`
127171

128172
### collection_funcs
129173

@@ -140,9 +184,11 @@
140184
- [x] ifnull
141185
- [ ] nanvl
142186
- [x] nullif
187+
- [ ] nullifzero
143188
- [x] nvl
144189
- [x] nvl2
145190
- [ ] when
191+
- [ ] zeroifnull
146192

147193
### conversion_funcs
148194

@@ -172,6 +218,7 @@
172218
- [ ] convert_timezone
173219
- [x] curdate
174220
- [x] current_date
221+
- [ ] current_time
175222
- [ ] current_timestamp
176223
- [x] current_timezone
177224
- [ ] date_add
@@ -185,6 +232,7 @@
185232
- [ ] datediff
186233
- [x] datepart
187234
- [ ] day
235+
- [ ] dayname
188236
- [ ] dayofmonth
189237
- [ ] dayofweek
190238
- [ ] dayofyear
@@ -197,27 +245,37 @@
197245
- [ ] make_date
198246
- [ ] make_dt_interval
199247
- [ ] make_interval
248+
- [ ] make_time
200249
- [ ] make_timestamp
201250
- [ ] make_timestamp_ltz
202251
- [ ] make_timestamp_ntz
203252
- [ ] make_ym_interval
204253
- [ ] minute
205254
- [ ] month
255+
- [ ] monthname
206256
- [ ] months_between
207257
- [ ] next_day
208258
- [ ] now
209259
- [ ] quarter
210260
- [ ] second
261+
- [ ] session_window
262+
- [ ] time_diff
263+
- [ ] time_trunc
211264
- [ ] timestamp_micros
212265
- [ ] timestamp_millis
213266
- [x] timestamp_seconds
214267
- [ ] to_date
268+
- [ ] to_time
215269
- [ ] to_timestamp
216270
- [ ] to_timestamp_ltz
217271
- [ ] to_timestamp_ntz
218272
- [ ] to_unix_timestamp
219273
- [ ] to_utc_timestamp
220274
- [ ] trunc
275+
- [ ] try_make_interval
276+
- [ ] try_make_timestamp
277+
- [ ] try_to_date
278+
- [ ] try_to_time
221279
- [ ] try_to_timestamp
222280
- [ ] unix_date
223281
- [ ] unix_micros
@@ -226,6 +284,8 @@
226284
- [x] unix_timestamp
227285
- [ ] weekday
228286
- [ ] weekofyear
287+
- [ ] window
288+
- [ ] window_time
229289
- [ ] year
230290

231291
### generator_funcs
@@ -289,11 +349,11 @@
289349

290350
### math_funcs
291351

292-
- [x] %
293-
- [x] -
294-
- [x] -
295-
- [x] -
296-
- [x] /
352+
- [x] `%`
353+
- [ ] `*`
354+
- [ ] `+`
355+
- [x] `-`
356+
- [x] `/`
297357
- [x] abs
298358
- [x] acos
299359
- [ ] acosh
@@ -339,6 +399,7 @@
339399
- [ ] rand
340400
- [ ] randn
341401
- [ ] random
402+
- [ ] randstr
342403
- [ ] rint
343404
- [x] round
344405
- [ ] sec
@@ -352,45 +413,84 @@
352413
- [ ] tanh
353414
- [x] try_add
354415
- [x] try_divide
416+
- [ ] try_mod
355417
- [x] try_multiply
356418
- [x] try_subtract
357419
- [x] unhex
420+
- [ ] uniform
358421
- [x] width_bucket
359422

360423
### misc_funcs
361424

362425
- [ ] aes_decrypt
363426
- [ ] aes_encrypt
427+
- [ ] approx_top_k_estimate
364428
- [ ] assert_true
429+
- [ ] bitmap_and_agg
430+
- [ ] bitmap_bit_position
431+
- [ ] bitmap_bucket_number
432+
- [ ] bitmap_construct_agg
433+
- [ ] bitmap_count
434+
- [ ] bitmap_or_agg
365435
- [x] current_catalog
366436
- [x] current_database
367437
- [x] current_schema
368438
- [x] current_user
369439
- [x] equal_null
440+
- [ ] from_avro
441+
- [ ] from_protobuf
442+
- [ ] hll_sketch_estimate
443+
- [ ] hll_union
370444
- [ ] input_file_block_length
371445
- [ ] input_file_block_start
372446
- [ ] input_file_name
447+
- [ ] is_variant_null
448+
- [ ] java_method
373449
- [x] monotonically_increasing_id
450+
- [ ] parse_json
374451
- [ ] raise_error
375452
- [x] rand
376453
- [x] randn
454+
- [ ] reflect
455+
- [ ] schema_of_avro
456+
- [ ] schema_of_variant
457+
- [ ] schema_of_variant_agg
458+
- [ ] session_user
377459
- [x] spark_partition_id
460+
- [ ] st_asbinary
461+
- [ ] st_geogfromwkb
462+
- [ ] st_geomfromwkb
463+
- [ ] st_setsrid
464+
- [ ] st_srid
465+
- [ ] theta_difference
466+
- [ ] theta_intersection
467+
- [ ] theta_sketch_estimate
468+
- [ ] theta_union
469+
- [ ] to_avro
470+
- [ ] to_protobuf
471+
- [ ] to_variant_object
472+
- [ ] try_aes_decrypt
473+
- [ ] try_parse_json
474+
- [ ] try_reflect
475+
- [ ] try_variant_get
378476
- [ ] typeof
379477
- [x] user
380478
- [ ] uuid
479+
- [ ] variant_get
381480
- [ ] version
382481

383482
### predicate_funcs
384483

385-
- [x] !
386-
- [x] <
387-
- [x] <=
388-
- [x] <=>
389-
- [x] =
390-
- [x] ==
391-
- [x] >
392-
- [x] > =
484+
- [x] `!`
485+
- [x] `<`
486+
- [x] `<=`
487+
- [x] `<=>`
488+
- [x] `=`
489+
- [x] `==`
490+
- [x] `>`
491+
- [x] `>=`
393492
- [x] and
493+
- [ ] between
394494
- [x] ilike
395495
- [x] in
396496
- [ ] isnan
@@ -413,6 +513,8 @@
413513
- [x] char_length
414514
- [x] character_length
415515
- [x] chr
516+
- [ ] collate
517+
- [ ] collation
416518
- [x] concat_ws
417519
- [x] contains
418520
- [ ] decode
@@ -424,6 +526,7 @@
424526
- [ ] format_string
425527
- [x] initcap
426528
- [x] instr
529+
- [ ] is_valid_utf8
427530
- [x] lcase
428531
- [ ] left
429532
- [x] len
@@ -433,11 +536,14 @@
433536
- [x] lower
434537
- [x] lpad
435538
- [x] ltrim
539+
- [ ] luhn_check
540+
- [ ] make_valid_utf8
436541
- [ ] mask
437542
- [x] octet_length
438543
- [ ] overlay
439544
- [ ] position
440545
- [ ] printf
546+
- [ ] quote
441547
- [ ] regexp_count
442548
- [ ] regexp_extract
443549
- [ ] regexp_extract_all
@@ -461,13 +567,16 @@
461567
- [ ] to_binary
462568
- [ ] to_char
463569
- [ ] to_number
570+
- [ ] to_varchar
464571
- [x] translate
465572
- [x] trim
466573
- [ ] try_to_binary
467574
- [ ] try_to_number
575+
- [ ] try_validate_utf8
468576
- [x] ucase
469577
- [ ] unbase64
470578
- [x] upper
579+
- [ ] validate_utf8
471580

472581
### struct_funcs
473582

@@ -477,6 +586,8 @@
477586
### url_funcs
478587

479588
- [ ] parse_url
589+
- [ ] try_parse_url
590+
- [ ] try_url_decode
480591
- [ ] url_decode
481592
- [ ] url_encode
482593

@@ -494,6 +605,9 @@
494605

495606
### xml_funcs
496607

608+
- [ ] from_xml
609+
- [ ] schema_of_xml
610+
- [ ] to_xml
497611
- [ ] xpath
498612
- [ ] xpath_boolean
499613
- [ ] xpath_double

0 commit comments

Comments
 (0)