Map array functions to ClickHouse equivalents

Add mappings for most Postgres array functions and operators to push
down to ClickHouse equivalents. Mark those without equivalents as
unshippable so that Postgres does not try to push them down and trigger
errors.

Also support pushing down array slices to the ClickHouse `arraySlice()`
function.

Refactor `classify_builtin_operator()` to work with operator OIDs rather
than strings, since the same operator string can be used for different
data types and thus would map to different ClickHouse functions.

Document the function mappings and add a new doc section for pushing
down operators that includes both the array operator mappings introduced
here as well as the pushed down regular expression operators added in
c047a7b & 3e765a6 and the JSON element extraction operators added in
0b4c03e.

While at it, fix some types in the docs.

Author: Philip Dubé <philip.dube@clickhouse.com>
Reviewed-by: David E. Wheeler <david.wheeler@clickhouse.com>

Author: serprex

CHANGELOG.md

@@ -28,6 +28,14 @@ All notable changes to this project will be documented in this file. It uses the
     prepending them to the regular expression (e.g., `(?i)foo`).
 *   Added pushdown for `regexp_split_to_array()` to `splitByRegexp()`,
     including flags.
+*   Added pushdown mappings for array functions: `array_cat`, `array_append`,
+    `array_remove`, `array_to_string`, `cardinality`, `array_length`,
+    `array_prepend`, `string_to_array`, `trim_array`, `array_fill`,
+    `array_reverse`, `array_shuffle`, `array_sample`, `array_sort`.
+*   Added pushdown for array operators: `@>` (`hasAll`), `<@` (`hasAll`),
+    `&&` (`hasAny`).
+*   Array slice syntax (`arr[L:U]`, `arr[:U]`, `arr[L:]`) now pushes down
+    as `arraySlice()`.
 
 ### ⬆️ Dependency Updates
 
@@ -39,6 +47,10 @@ All notable changes to this project will be documented in this file. It uses the
     unable to map a ClickHouse type to a Postgres type.
 *   Fixed reversal of the arguments passed to the ClickHouse `match()`
     function by the mapping from `regexp_like()`.
+*   `array_dims`, `array_ndims`, `array_lower`, `array_upper`, `array_replace`,
+    `array_positions`, `array_fill (3-arg)`, `array_sort (3-arg)`, and
+    `string_to_array(3-arg)` now evaluate locally instead of being pushed to
+    ClickHouse where they would fail.
 
   [v0.1.11]: https://github.com/ClickHouse/pg_clickhouse/compare/v0.1.10...v0.1.11
 

doc/pg_clickhouse.md

@@ -287,13 +287,13 @@ Use [CREATE FOREIGN TABLE] to create a foreign table that can query data from
 a ClickHouse database:
 
 ```sql
-CREATE FOREIGN TABLE uact (
+CREATE FOREIGN TABLE acts (
     user_id    bigint NOT NULL,
     page_views int,
     duration   smallint,
     sign       smallint
 ) SERVER taxi_srv OPTIONS(
-    table_name 'uact',
+    table_name 'acts',
     engine 'CollapsingMergeTree'
 );
 ```
@@ -325,8 +325,6 @@ option:
 
 Example:
 
-(aggregatefunction 'sum')
-
 ```sql
 CREATE FOREIGN TABLE test (
     column1 bigint  OPTIONS(AggregateFunction 'uniq'),
@@ -354,14 +352,14 @@ TABLE].
 Use [DROP FOREIGN TABLE] to remove a foreign table:
 
 ```sql
-DROP FOREIGN TABLE uact;
+DROP FOREIGN TABLE acts;
 ```
 
 This command fails if there are any objects that depend on the foreign table.
 Use the `CASCADE` clause to drop them, too:
 
 ```sql
-DROP FOREIGN TABLE uact CASCADE;
+DROP FOREIGN TABLE acts CASCADE;
 ```
 
 ## DML SQL Reference
@@ -422,7 +420,7 @@ like any other tables:
 try=# SELECT start_at, duration, resource FROM logs WHERE req_id = 4117909262;
           start_at          | duration |    resource
 ----------------------------+----------+----------------
- 2025-12-05 15:07:32.944188 |      175 | /widgets/totam
+ 2025-12-05 15:07:32.944188 |      175 | /widgets/totem
 (1 row)
 ```
 
@@ -922,7 +920,7 @@ SELECT c1, encode(c2::bytea, 'hex'), encode(c3::bytea, 'hex') FROM texts ORDER B
 
 The text columns will be correct:
 
-```pgdsql
+```pgsql
 
  c1 |                          encode                          |              encode
 ----+----------------------------------------------------------+----------------------------------
@@ -1029,6 +1027,19 @@ maps the following functions:
     *   `date_trunc('quarter')`: [toStartOfQuarter](https://clickhouse.com/docs/sql-reference/functions/date-time-functions#toStartOfQuarter)
     *   `date_trunc('year')`: [toStartOfYear](https://clickhouse.com/docs/sql-reference/functions/date-time-functions#toStartOfYear)
 *   `array_position`: [indexOf](https://clickhouse.com/docs/sql-reference/functions/array-functions#indexOf)
+*   `array_cat`: [arrayConcat](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayConcat)
+*   `array_append`: [arrayPushBack](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayPushBack)
+*   `array_prepend`: [arrayPushFront](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayPushFront)
+*   `array_remove`: [arrayRemove](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayRemove)
+*   `array_length` & `cardinality`: [length](https://clickhouse.com/docs/sql-reference/functions/array-functions#length)
+*   `array_to_string`: [arrayStringConcat](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayStringConcat)
+*   `string_to_array`: [splitByString](https://clickhouse.com/docs/sql-reference/functions/splitting-merging-functions#splitByString)
+*   `trim_array`: [arrayResize](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayResize)
+*   `array_fill`: [arrayWithConstant](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayWithConstant)
+*   `array_reverse`: [arrayReverse](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayReverse)
+*   `array_shuffle`: [arrayShuffle](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayShuffle)
+*   `array_sample`: [arrayRandomSample](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayRandomSample)
+*   `array_sort`: [arraySort](https://clickhouse.com/docs/sql-reference/functions/array-functions#arraySort) / [arrayReverseSort](https://clickhouse.com/docs/sql-reference/functions/array-functions#arrayReverseSort)
 *   `btrim`: [trimBoth](https://clickhouse.com/docs/sql-reference/functions/string-functions#trimboth)
 *   `strpos`: [position](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#position)
 *   `regexp_like`: [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match)
@@ -1056,6 +1067,19 @@ maps the following functions:
 *   `CURRENT_ROLE`: Passed as value from PostgreSQL function.
 *   `SESSION_USER`: Passed as value from PostgreSQL function.
 
+### Pushdown Operators
+
+*   Array slice (`arr[L:U]`): [arraySlice](https://clickhouse.com/docs/sql-reference/functions/array-functions#arraySlice)
+*   `@>` (array contains): [hasAll](https://clickhouse.com/docs/sql-reference/functions/array-functions#hasAll)
+*   `<@` (array contained by): [hasAll](https://clickhouse.com/docs/sql-reference/functions/array-functions#hasAll)
+*   `&&` (array overlap): [hasAny](https://clickhouse.com/docs/sql-reference/functions/array-functions#hasAny)
+*   `~` (regexp match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match)
+*   `!~` (regexp not match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match)
+*   `~*` (case insensitive regexp no match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match)
+*   `!~*` (case insensitive regexp not match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match)
+*   `->` (JSON extract element): [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns)
+*   `->>` (JSON extract element as text): [toJSONString](https://clickhouse.com/docs/sql-reference/functions/json-functions#toJSONString) + [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns)
+
 ### Custom Functions
 
 These custom functions created by `pg_clickhouse` provide foreign query

src/custom_types.c

@@ -4,6 +4,7 @@
 #include "access/htup.h"
 #include "access/htup_details.h"
 #include "catalog/dependency.h"
+#include "catalog/pg_operator_d.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
 #include "commands/defrem.h"
@@ -50,12 +51,32 @@
 #define F_EXTRACT_TEXT_TIMESTAMP 6202
 #define F_EXTRACT_TEXT_TIMESTAMPTZ 6203
 #define F_EXTRACT_TEXT_DATE 6199
+#define F_TRIM_ARRAY 6172
+#define F_STRING_TO_ARRAY_TEXT_TEXT 394
+#define F_STRING_TO_ARRAY_TEXT_TEXT_TEXT 376
+#define F_ARRAY_TO_STRING_ANYARRAY_TEXT 395
+#define F_ARRAY_TO_STRING_ANYARRAY_TEXT_TEXT 384
+#define F_ARRAY_FILL_ANYELEMENT__INT4 F_ARRAY_FILL
+#define F_ARRAY_FILL_ANYELEMENT__INT4__INT4 F_ARRAY_FILL_WITH_LOWER_BOUNDS
+#define F_CARDINALITY F_ARRAY_CARDINALITY
 #endif
-/* regexp_like was added in Postgres 15; Mock it for earlier versions. */
+/* regexp_like was added in Postgres 15 */
 #if PG_VERSION_NUM < 150000
 #define F_REGEXP_LIKE_TEXT_TEXT 6263
 #define F_REGEXP_LIKE_TEXT_TEXT_TEXT 6264
 #endif
+/* array_shuffle, array_sample added in Postgres 16 */
+#if PG_VERSION_NUM < 160000
+#define F_ARRAY_SHUFFLE 6215
+#define F_ARRAY_SAMPLE 6216
+#endif
+/* array_reverse, array_sort added in Postgres 18 */
+#if PG_VERSION_NUM < 180000
+#define F_ARRAY_REVERSE 6381
+#define F_ARRAY_SORT_ANYARRAY 6388
+#define F_ARRAY_SORT_ANYARRAY_BOOL 6389
+#define F_ARRAY_SORT_ANYARRAY_BOOL_BOOL 6390
+#endif
 
 #define STR_STARTS_WITH(str, sub) strncmp(str, sub, strlen(sub)) == 0
 #define STR_EQUAL(a, b) strcmp(a, b) == 0
@@ -217,6 +238,34 @@ chfdw_check_for_custom_function(Oid funcid)
 			case F_CLOCK_TIMESTAMP:
 			case F_CURRENT_SCHEMA:
 			case F_CURRENT_DATABASE:
+				/* array functions: simple mappings */
+			case F_ARRAY_CAT:
+			case F_ARRAY_APPEND:
+			case F_ARRAY_REMOVE:
+			case F_ARRAY_TO_STRING_ANYARRAY_TEXT:
+			case F_CARDINALITY:
+			case F_ARRAY_REVERSE:
+			case F_ARRAY_SORT_ANYARRAY:
+			case F_ARRAY_SHUFFLE:
+			case F_ARRAY_SAMPLE:
+				/* array functions: arg rewriting */
+			case F_ARRAY_LENGTH:
+			case F_ARRAY_PREPEND:
+			case F_STRING_TO_ARRAY_TEXT_TEXT:
+			case F_TRIM_ARRAY:
+			case F_ARRAY_FILL_ANYELEMENT__INT4:
+			case F_ARRAY_SORT_ANYARRAY_BOOL:
+				/* array functions: unshippable */
+			case F_ARRAY_DIMS:
+			case F_ARRAY_NDIMS:
+			case F_ARRAY_LOWER:
+			case F_ARRAY_UPPER:
+			case F_ARRAY_REPLACE:
+			case F_ARRAY_POSITIONS:
+			case F_ARRAY_TO_STRING_ANYARRAY_TEXT_TEXT:
+			case F_STRING_TO_ARRAY_TEXT_TEXT_TEXT:
+			case F_ARRAY_FILL_ANYELEMENT__INT4__INT4:
+			case F_ARRAY_SORT_ANYARRAY_BOOL_BOOL:
 				special_builtin = true;
 				break;
 			default:
@@ -363,6 +412,69 @@ chfdw_check_for_custom_function(Oid funcid)
 					entry->custom_name[0] = '\1';
 					break;
 				}
+				/* array functions: unshippable */
+			case F_ARRAY_DIMS:
+			case F_ARRAY_NDIMS:
+			case F_ARRAY_LOWER:
+			case F_ARRAY_UPPER:
+			case F_ARRAY_REPLACE:
+			case F_ARRAY_POSITIONS:
+			case F_ARRAY_TO_STRING_ANYARRAY_TEXT_TEXT:
+			case F_STRING_TO_ARRAY_TEXT_TEXT_TEXT:
+			case F_ARRAY_FILL_ANYELEMENT__INT4__INT4:
+			case F_ARRAY_SORT_ANYARRAY_BOOL_BOOL:
+				entry->cf_type = CF_UNSHIPPABLE;
+				break;
+				/* array functions: simple mappings */
+			case F_ARRAY_CAT:
+				strcpy(entry->custom_name, "arrayConcat");
+				break;
+			case F_ARRAY_APPEND:
+				strcpy(entry->custom_name, "arrayPushBack");
+				break;
+			case F_ARRAY_REMOVE:
+				strcpy(entry->custom_name, "arrayRemove");
+				break;
+			case F_ARRAY_TO_STRING_ANYARRAY_TEXT:
+				strcpy(entry->custom_name, "arrayStringConcat");
+				break;
+			case F_CARDINALITY:
+			case F_ARRAY_LENGTH:
+				entry->cf_type = CF_ARRAY_LENGTH;
+				strcpy(entry->custom_name, "length");
+				break;
+			case F_ARRAY_REVERSE:
+				strcpy(entry->custom_name, "arrayReverse");
+				break;
+			case F_ARRAY_SORT_ANYARRAY:
+				strcpy(entry->custom_name, "arraySort");
+				break;
+			case F_ARRAY_SHUFFLE:
+				strcpy(entry->custom_name, "arrayShuffle");
+				break;
+			case F_ARRAY_SAMPLE:
+				strcpy(entry->custom_name, "arrayRandomSample");
+				break;
+			case F_ARRAY_PREPEND:
+				entry->cf_type = CF_ARRAY_PREPEND;
+				strcpy(entry->custom_name, "arrayPushFront");
+				break;
+			case F_STRING_TO_ARRAY_TEXT_TEXT:
+				entry->cf_type = CF_STRING_TO_ARRAY;
+				strcpy(entry->custom_name, "splitByString");
+				break;
+			case F_TRIM_ARRAY:
+				entry->cf_type = CF_TRIM_ARRAY;
+				strcpy(entry->custom_name, "arrayResize");
+				break;
+			case F_ARRAY_FILL_ANYELEMENT__INT4:
+				entry->cf_type = CF_ARRAY_FILL;
+				strcpy(entry->custom_name, "arrayWithConstant");
+				break;
+			case F_ARRAY_SORT_ANYARRAY_BOOL:
+				entry->cf_type = CF_ARRAY_SORT_DESC;
+				entry->custom_name[0] = '\1';
+				break;
 		}
 
 		if (special_builtin)
@@ -426,39 +538,42 @@ chfdw_check_for_custom_type(Oid typeoid)
 	return entry;
 }
 
-/*
- * Operator-name to custom_object_type mapping table, searched linearly by
- * classify_builtin_operator().  Keep in sync with the CF_* enum in fdw.h.
- */
-typedef struct
-{
-	const char *oprname;
-	custom_object_type ctype;
-}			BuiltinOperatorMap;
-
-static const BuiltinOperatorMap builtin_operator_map[] = {
-	{"~", CF_REGEX_MATCH},
-	{"!~", CF_REGEX_NO_MATCH},
-	{"~*", CF_REGEX_ICASE_MATCH},
-	{"!~*", CF_REGEX_ICASE_NO_MATCH},
-	{"->", CF_JSONB_FETCHVAL},
-	{"->>", CF_JSONB_FETCHVAL_TEXT},
-};
+/* pg_operator_d.h only has oid_symbol for some operators */
+#define OID_TEXT_REGEX_NE_OP		642
+#define OID_TEXT_IREGEX_NE_OP		1229
+#define OID_JSONB_FETCHVAL_OP		3211
+#define OID_JSONB_FETCHVAL_TEXT_OP	3477
 
 /*
- * Map a builtin operator name to its custom_object_type.  Returns CF_USUAL
- * when the operator needs no special handling and should follow the normal
- * builtin shortcut (i.e. be presumed shippable with no rewrite).
+ * Map a builtin operator OID to its custom_object_type. Returns CF_USUAL
+ * when the operator needs no special handling.
  */
 static custom_object_type
-classify_builtin_operator(const char *oprname)
+classify_builtin_operator(Oid opoid)
 {
-	for (int i = 0; i < lengthof(builtin_operator_map); i++)
+	switch (opoid)
 	{
-		if (strcmp(oprname, builtin_operator_map[i].oprname) == 0)
-			return builtin_operator_map[i].ctype;
+		case OID_TEXT_REGEXEQ_OP:
+			return CF_REGEX_MATCH;
+		case OID_TEXT_REGEX_NE_OP:
+			return CF_REGEX_NO_MATCH;
+		case OID_TEXT_ICREGEXEQ_OP:
+			return CF_REGEX_ICASE_MATCH;
+		case OID_TEXT_IREGEX_NE_OP:
+			return CF_REGEX_ICASE_NO_MATCH;
+		case OID_JSONB_FETCHVAL_OP:
+			return CF_JSONB_FETCHVAL;
+		case OID_JSONB_FETCHVAL_TEXT_OP:
+			return CF_JSONB_FETCHVAL_TEXT;
+		case OID_ARRAY_CONTAINS_OP:
+			return CF_ARRAY_CONTAINS;
+		case OID_ARRAY_CONTAINED_OP:
+			return CF_ARRAY_CONTAINED_BY;
+		case OID_ARRAY_OVERLAP_OP:
+			return CF_ARRAY_OVERLAP;
+		default:
+			return CF_USUAL;
 	}
-	return CF_USUAL;
 }
 
 CustomObjectDef *
@@ -474,31 +589,10 @@ chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form)
 
 	if (chfdw_is_builtin(opoid))
 	{
-		switch (opoid)
+		ctype = classify_builtin_operator(opoid);
+		if (ctype == CF_USUAL && opoid != F_TIMESTAMPTZ_PL_INTERVAL)
 		{
-				/* timestamptz + interval */
-			case F_TIMESTAMPTZ_PL_INTERVAL:
-				break;
-			default:
-
-				/* Look up the operator name so we can classify it. */
-				if (!form)
-				{
-					tuple = SearchSysCache1(OPEROID, ObjectIdGetDatum(opoid));
-					if (!HeapTupleIsValid(tuple))
-						ereport(ERROR,
-								errcode(ERRCODE_INTERNAL_ERROR),
-								errmsg("pg_clickhouse: cache lookup failed for operator %u", opoid));
-					form = (Form_pg_operator) GETSTRUCT(tuple);
-				}
-
-				ctype = classify_builtin_operator(NameStr(form->oprname));
-				if (ctype != CF_USUAL)
-					break;		/* fall through to cache + classify below */
-
-				if (tuple)
-					ReleaseSysCache(tuple);
-				return NULL;
+			return NULL;
 		}
 	}
 
@@ -516,10 +610,11 @@ chfdw_check_for_custom_operator(Oid opoid, Form_pg_operator form)
 		entry = hash_search(custom_objects_cache, (void *) &opoid, HASH_ENTER, NULL);
 		init_custom_entry(entry);
 
+		ctype = classify_builtin_operator(opoid);
 		if (opoid == F_TIMESTAMPTZ_PL_INTERVAL)
 			entry->cf_type = CF_TIMESTAMPTZ_PL_INTERVAL;
-		else if (form && classify_builtin_operator(NameStr(form->oprname)) != CF_USUAL)
-			entry->cf_type = classify_builtin_operator(NameStr(form->oprname));
+		else if (ctype != CF_USUAL)
+			entry->cf_type = ctype;
 		else
 		{
 			Oid			extoid = getExtensionOfObject(OperatorRelationId, opoid);