|
19 | 19 |
|
20 | 20 | # Supported Spark Expressions |
21 | 21 |
|
| 22 | +## How to Read This Document |
| 23 | + |
| 24 | +- A function marked with `[x]` has a native implementation in Comet and does not fall back to Spark by default. |
| 25 | +- A function marked with `[ ]` has no native Comet implementation and falls back to Spark. |
| 26 | + |
| 27 | +> **Note:** Some functions listed as supported may still be incompatible with Spark in |
| 28 | +> certain cases (data types, modes, edge values) and fall back to Spark at runtime. Full |
| 29 | +> per-function details are documented in the |
| 30 | +> [Compatibility Guide](https://datafusion.apache.org/comet/user-guide/latest/compatibility/expressions/index.html). |
| 31 | +
|
22 | 32 | ### agg_funcs |
23 | 33 |
|
24 | 34 | - [x] any |
25 | 35 | - [x] any_value |
26 | 36 | - [ ] approx_count_distinct |
27 | 37 | - [ ] approx_percentile |
| 38 | +- [ ] approx_top_k |
| 39 | +- [ ] approx_top_k_accumulate |
| 40 | +- [ ] approx_top_k_combine |
28 | 41 | - [ ] array_agg |
29 | 42 | - [x] avg |
30 | 43 | - [x] bit_and |
|
46 | 59 | - [ ] grouping |
47 | 60 | - [ ] grouping_id |
48 | 61 | - [ ] histogram_numeric |
| 62 | +- [ ] hll_sketch_agg |
| 63 | +- [ ] hll_union_agg |
| 64 | +- [ ] kll_sketch_agg_bigint |
| 65 | +- [ ] kll_sketch_agg_double |
| 66 | +- [ ] kll_sketch_agg_float |
| 67 | +- [ ] kll_sketch_get_n_bigint |
| 68 | +- [ ] kll_sketch_get_n_double |
| 69 | +- [ ] kll_sketch_get_n_float |
| 70 | +- [ ] kll_sketch_get_quantile_bigint |
| 71 | +- [ ] kll_sketch_get_quantile_double |
| 72 | +- [ ] kll_sketch_get_quantile_float |
| 73 | +- [ ] kll_sketch_get_rank_bigint |
| 74 | +- [ ] kll_sketch_get_rank_double |
| 75 | +- [ ] kll_sketch_get_rank_float |
| 76 | +- [ ] kll_sketch_merge_bigint |
| 77 | +- [ ] kll_sketch_merge_double |
| 78 | +- [ ] kll_sketch_merge_float |
| 79 | +- [ ] kll_sketch_to_string_bigint |
| 80 | +- [ ] kll_sketch_to_string_double |
| 81 | +- [ ] kll_sketch_to_string_float |
49 | 82 | - [ ] kurtosis |
50 | 83 | - [x] last |
51 | 84 | - [x] last_value |
| 85 | +- [ ] listagg |
52 | 86 | - [x] max |
53 | 87 | - [ ] max_by |
54 | 88 | - [x] mean |
|
58 | 92 | - [ ] mode |
59 | 93 | - [ ] percentile |
60 | 94 | - [ ] percentile_approx |
| 95 | +- [ ] percentile_cont |
| 96 | +- [ ] percentile_disc |
61 | 97 | - [x] regr_avgx |
62 | 98 | - [x] regr_avgy |
63 | 99 | - [x] regr_count |
|
73 | 109 | - [x] stddev |
74 | 110 | - [x] stddev_pop |
75 | 111 | - [x] stddev_samp |
| 112 | +- [ ] string_agg |
76 | 113 | - [x] sum |
| 114 | +- [ ] theta_intersection_agg |
| 115 | +- [ ] theta_sketch_agg |
| 116 | +- [ ] theta_union_agg |
77 | 117 | - [ ] try_avg |
78 | 118 | - [ ] try_sum |
79 | 119 | - [x] var_pop |
|
100 | 140 | - [x] array_max |
101 | 141 | - [ ] array_min |
102 | 142 | - [x] array_position |
| 143 | +- [ ] array_prepend |
103 | 144 | - [x] array_remove |
104 | 145 | - [x] array_repeat |
105 | 146 | - [x] array_union |
|
115 | 156 |
|
116 | 157 | ### bitwise_funcs |
117 | 158 |
|
118 | | -- [x] & |
119 | | -- [x] ^ |
| 159 | +- [x] `&` |
| 160 | +- [ ] `<<` |
| 161 | +- [ ] `>>` |
| 162 | +- [ ] `>>>` |
| 163 | +- [x] `^` |
120 | 164 | - [ ] bit_count |
121 | 165 | - [ ] bit_get |
122 | 166 | - [ ] getbit |
123 | 167 | - [x] shiftright |
124 | 168 | - [ ] shiftrightunsigned |
125 | | -- [x] | |
126 | | -- [x] ~ |
| 169 | +- [x] `|` |
| 170 | +- [x] `~` |
127 | 171 |
|
128 | 172 | ### collection_funcs |
129 | 173 |
|
|
140 | 184 | - [x] ifnull |
141 | 185 | - [ ] nanvl |
142 | 186 | - [x] nullif |
| 187 | +- [ ] nullifzero |
143 | 188 | - [x] nvl |
144 | 189 | - [x] nvl2 |
145 | 190 | - [ ] when |
| 191 | +- [ ] zeroifnull |
146 | 192 |
|
147 | 193 | ### conversion_funcs |
148 | 194 |
|
|
172 | 218 | - [ ] convert_timezone |
173 | 219 | - [x] curdate |
174 | 220 | - [x] current_date |
| 221 | +- [ ] current_time |
175 | 222 | - [ ] current_timestamp |
176 | 223 | - [x] current_timezone |
177 | 224 | - [ ] date_add |
|
185 | 232 | - [ ] datediff |
186 | 233 | - [x] datepart |
187 | 234 | - [ ] day |
| 235 | +- [ ] dayname |
188 | 236 | - [ ] dayofmonth |
189 | 237 | - [ ] dayofweek |
190 | 238 | - [ ] dayofyear |
|
197 | 245 | - [ ] make_date |
198 | 246 | - [ ] make_dt_interval |
199 | 247 | - [ ] make_interval |
| 248 | +- [ ] make_time |
200 | 249 | - [ ] make_timestamp |
201 | 250 | - [ ] make_timestamp_ltz |
202 | 251 | - [ ] make_timestamp_ntz |
203 | 252 | - [ ] make_ym_interval |
204 | 253 | - [ ] minute |
205 | 254 | - [ ] month |
| 255 | +- [ ] monthname |
206 | 256 | - [ ] months_between |
207 | 257 | - [ ] next_day |
208 | 258 | - [ ] now |
209 | 259 | - [ ] quarter |
210 | 260 | - [ ] second |
| 261 | +- [ ] session_window |
| 262 | +- [ ] time_diff |
| 263 | +- [ ] time_trunc |
211 | 264 | - [ ] timestamp_micros |
212 | 265 | - [ ] timestamp_millis |
213 | 266 | - [x] timestamp_seconds |
214 | 267 | - [ ] to_date |
| 268 | +- [ ] to_time |
215 | 269 | - [ ] to_timestamp |
216 | 270 | - [ ] to_timestamp_ltz |
217 | 271 | - [ ] to_timestamp_ntz |
218 | 272 | - [ ] to_unix_timestamp |
219 | 273 | - [ ] to_utc_timestamp |
220 | 274 | - [ ] trunc |
| 275 | +- [ ] try_make_interval |
| 276 | +- [ ] try_make_timestamp |
| 277 | +- [ ] try_to_date |
| 278 | +- [ ] try_to_time |
221 | 279 | - [ ] try_to_timestamp |
222 | 280 | - [ ] unix_date |
223 | 281 | - [ ] unix_micros |
|
226 | 284 | - [x] unix_timestamp |
227 | 285 | - [ ] weekday |
228 | 286 | - [ ] weekofyear |
| 287 | +- [ ] window |
| 288 | +- [ ] window_time |
229 | 289 | - [ ] year |
230 | 290 |
|
231 | 291 | ### generator_funcs |
|
289 | 349 |
|
290 | 350 | ### math_funcs |
291 | 351 |
|
292 | | -- [x] % |
293 | | -- [x] - |
294 | | -- [x] - |
295 | | -- [x] - |
296 | | -- [x] / |
| 352 | +- [x] `%` |
| 353 | +- [ ] `*` |
| 354 | +- [ ] `+` |
| 355 | +- [x] `-` |
| 356 | +- [x] `/` |
297 | 357 | - [x] abs |
298 | 358 | - [x] acos |
299 | 359 | - [ ] acosh |
|
339 | 399 | - [ ] rand |
340 | 400 | - [ ] randn |
341 | 401 | - [ ] random |
| 402 | +- [ ] randstr |
342 | 403 | - [ ] rint |
343 | 404 | - [x] round |
344 | 405 | - [ ] sec |
|
352 | 413 | - [ ] tanh |
353 | 414 | - [x] try_add |
354 | 415 | - [x] try_divide |
| 416 | +- [ ] try_mod |
355 | 417 | - [x] try_multiply |
356 | 418 | - [x] try_subtract |
357 | 419 | - [x] unhex |
| 420 | +- [ ] uniform |
358 | 421 | - [x] width_bucket |
359 | 422 |
|
360 | 423 | ### misc_funcs |
361 | 424 |
|
362 | 425 | - [ ] aes_decrypt |
363 | 426 | - [ ] aes_encrypt |
| 427 | +- [ ] approx_top_k_estimate |
364 | 428 | - [ ] assert_true |
| 429 | +- [ ] bitmap_and_agg |
| 430 | +- [ ] bitmap_bit_position |
| 431 | +- [ ] bitmap_bucket_number |
| 432 | +- [ ] bitmap_construct_agg |
| 433 | +- [ ] bitmap_count |
| 434 | +- [ ] bitmap_or_agg |
365 | 435 | - [x] current_catalog |
366 | 436 | - [x] current_database |
367 | 437 | - [x] current_schema |
368 | 438 | - [x] current_user |
369 | 439 | - [x] equal_null |
| 440 | +- [ ] from_avro |
| 441 | +- [ ] from_protobuf |
| 442 | +- [ ] hll_sketch_estimate |
| 443 | +- [ ] hll_union |
370 | 444 | - [ ] input_file_block_length |
371 | 445 | - [ ] input_file_block_start |
372 | 446 | - [ ] input_file_name |
| 447 | +- [ ] is_variant_null |
| 448 | +- [ ] java_method |
373 | 449 | - [x] monotonically_increasing_id |
| 450 | +- [ ] parse_json |
374 | 451 | - [ ] raise_error |
375 | 452 | - [x] rand |
376 | 453 | - [x] randn |
| 454 | +- [ ] reflect |
| 455 | +- [ ] schema_of_avro |
| 456 | +- [ ] schema_of_variant |
| 457 | +- [ ] schema_of_variant_agg |
| 458 | +- [ ] session_user |
377 | 459 | - [x] spark_partition_id |
| 460 | +- [ ] st_asbinary |
| 461 | +- [ ] st_geogfromwkb |
| 462 | +- [ ] st_geomfromwkb |
| 463 | +- [ ] st_setsrid |
| 464 | +- [ ] st_srid |
| 465 | +- [ ] theta_difference |
| 466 | +- [ ] theta_intersection |
| 467 | +- [ ] theta_sketch_estimate |
| 468 | +- [ ] theta_union |
| 469 | +- [ ] to_avro |
| 470 | +- [ ] to_protobuf |
| 471 | +- [ ] to_variant_object |
| 472 | +- [ ] try_aes_decrypt |
| 473 | +- [ ] try_parse_json |
| 474 | +- [ ] try_reflect |
| 475 | +- [ ] try_variant_get |
378 | 476 | - [ ] typeof |
379 | 477 | - [x] user |
380 | 478 | - [ ] uuid |
| 479 | +- [ ] variant_get |
381 | 480 | - [ ] version |
382 | 481 |
|
383 | 482 | ### predicate_funcs |
384 | 483 |
|
385 | | -- [x] ! |
386 | | -- [x] < |
387 | | -- [x] <= |
388 | | -- [x] <=> |
389 | | -- [x] = |
390 | | -- [x] == |
391 | | -- [x] > |
392 | | -- [x] > = |
| 484 | +- [x] `!` |
| 485 | +- [x] `<` |
| 486 | +- [x] `<=` |
| 487 | +- [x] `<=>` |
| 488 | +- [x] `=` |
| 489 | +- [x] `==` |
| 490 | +- [x] `>` |
| 491 | +- [x] `>=` |
393 | 492 | - [x] and |
| 493 | +- [ ] between |
394 | 494 | - [x] ilike |
395 | 495 | - [x] in |
396 | 496 | - [ ] isnan |
|
413 | 513 | - [x] char_length |
414 | 514 | - [x] character_length |
415 | 515 | - [x] chr |
| 516 | +- [ ] collate |
| 517 | +- [ ] collation |
416 | 518 | - [x] concat_ws |
417 | 519 | - [x] contains |
418 | 520 | - [ ] decode |
|
424 | 526 | - [ ] format_string |
425 | 527 | - [x] initcap |
426 | 528 | - [x] instr |
| 529 | +- [ ] is_valid_utf8 |
427 | 530 | - [x] lcase |
428 | 531 | - [ ] left |
429 | 532 | - [x] len |
|
433 | 536 | - [x] lower |
434 | 537 | - [x] lpad |
435 | 538 | - [x] ltrim |
| 539 | +- [ ] luhn_check |
| 540 | +- [ ] make_valid_utf8 |
436 | 541 | - [ ] mask |
437 | 542 | - [x] octet_length |
438 | 543 | - [ ] overlay |
439 | 544 | - [ ] position |
440 | 545 | - [ ] printf |
| 546 | +- [ ] quote |
441 | 547 | - [ ] regexp_count |
442 | 548 | - [ ] regexp_extract |
443 | 549 | - [ ] regexp_extract_all |
|
461 | 567 | - [ ] to_binary |
462 | 568 | - [ ] to_char |
463 | 569 | - [ ] to_number |
| 570 | +- [ ] to_varchar |
464 | 571 | - [x] translate |
465 | 572 | - [x] trim |
466 | 573 | - [ ] try_to_binary |
467 | 574 | - [ ] try_to_number |
| 575 | +- [ ] try_validate_utf8 |
468 | 576 | - [x] ucase |
469 | 577 | - [ ] unbase64 |
470 | 578 | - [x] upper |
| 579 | +- [ ] validate_utf8 |
471 | 580 |
|
472 | 581 | ### struct_funcs |
473 | 582 |
|
|
477 | 586 | ### url_funcs |
478 | 587 |
|
479 | 588 | - [ ] parse_url |
| 589 | +- [ ] try_parse_url |
| 590 | +- [ ] try_url_decode |
480 | 591 | - [ ] url_decode |
481 | 592 | - [ ] url_encode |
482 | 593 |
|
|
494 | 605 |
|
495 | 606 | ### xml_funcs |
496 | 607 |
|
| 608 | +- [ ] from_xml |
| 609 | +- [ ] schema_of_xml |
| 610 | +- [ ] to_xml |
497 | 611 | - [ ] xpath |
498 | 612 | - [ ] xpath_boolean |
499 | 613 | - [ ] xpath_double |
|
0 commit comments