support multidimensional arrays

needed for functions like extractAllGroupsHorizontal returning Array(Array(String))

Author: serprex

src/binary.cpp

@@ -326,8 +326,13 @@ extern "C"
 				return UUIDOID;
 			case Type::Code::Array:
 			{
-				Oid array_type
-				= get_array_type(get_corr_postgres_type(type->As<clickhouse::ArrayType>()->GetItemType()));
+				/* postgres uses one array type for any number of dimensions, so
+				 * walk past nested Array layers to the leaf element type. */
+				auto leaf = type->As<clickhouse::ArrayType>()->GetItemType();
+				while (leaf->GetCode() == Type::Code::Array)
+					leaf = leaf->As<clickhouse::ArrayType>()->GetItemType();
+
+				Oid array_type = get_array_type(get_corr_postgres_type(leaf));
 				if (array_type == InvalidOid)
 					throw std::runtime_error("pg_clickhouse: could not find array "
 											 " type for column type "
@@ -1075,14 +1080,25 @@ extern "C"
 				size_t len = arr->Size();
 				auto   slot = (ch_binary_array_t *)exc_palloc(sizeof(ch_binary_array_t));
 
-				Oid item_type = get_corr_postgres_type(arr->Type());
+				/* find leaf scalar type & nesting depth, since postgres has one
+				 * array type per element type regardless of ndim */
+				int	 ndim = 1;
+				auto leaf_type = arr->Type();
+				while (leaf_type->GetCode() == Type::Code::Array)
+				{
+					leaf_type = leaf_type->As<clickhouse::ArrayType>()->GetItemType();
+					ndim++;
+				}
+
+				Oid item_type = get_corr_postgres_type(leaf_type);
 				Oid array_type = get_array_type(item_type);
 
 				if (array_type == InvalidOid)
 					throw std::runtime_error(std::string("pg_clickhouse: could not") + " find array type for "
 											 + std::to_string(item_type));
 
 				slot->len = len;
+				slot->ndim = ndim;
 				slot->array_type = array_type;
 				slot->item_type = item_type;
 
@@ -1091,8 +1107,13 @@ extern "C"
 					slot->datums = (Datum *)exc_palloc0(sizeof(Datum) * len);
 					slot->nulls = (bool *)exc_palloc0(sizeof(bool) * len);
 
+					/* For ndim==1 inner make_datum returns leaf scalars; for
+					 * ndim>1 it recurses into the Array branch and produces
+					 * nested ch_binary_array_t* values. Use a scratch valtype
+					 * to avoid clobbering slot->item_type. */
+					Oid scratch;
 					for (size_t i = 0; i < len; ++i)
-						slot->datums[i] = make_datum(arr, i, &slot->item_type, &slot->nulls[i]);
+						slot->datums[i] = make_datum(arr, i, &scratch, &slot->nulls[i]);
 				}
 
 				/* this one will need additional work, since we just return raw slot */

src/convert.c

@@ -138,20 +138,160 @@ convert_out_generic(ch_convert_output_state * state, Datum val)
 	return val;
 }
 
+/*
+ * Walk nested ch_binary_array_t into a flat datum buffer, verifying each
+ * level matches the dims taken from the first child. Returns false if the
+ * shape is jagged so the caller can fall back to a slower path.
+ */
+static bool
+flatten_nested_array(ch_binary_array_t * slot, int *dims, int level,
+					 Datum * values, bool *nulls, size_t * idx)
+{
+	if ((int) slot->len != dims[level])
+		return false;
+
+	if (slot->ndim == 1)
+	{
+		for (size_t i = 0; i < slot->len; i++)
+		{
+			values[*idx] = slot->datums[i];
+			nulls[*idx] = slot->nulls[i];
+			(*idx)++;
+		}
+	}
+	else
+	{
+		for (size_t i = 0; i < slot->len; i++)
+		{
+			ch_binary_array_t *child = (ch_binary_array_t *) DatumGetPointer(slot->datums[i]);
+
+			if (!flatten_nested_array(child, dims, level + 1, values, nulls, idx))
+				return false;
+		}
+	}
+	return true;
+}
+
+/*
+ * Emit a nested ch_binary_array_t as a postgres array text literal, quoting
+ * each leaf and escaping `\` and `"`. Used as the jagged fallback so binary
+ * surfaces the same array_in malformed-literal error as the http path.
+ */
+static void
+emit_nested_array_text(ch_binary_array_t * slot, FmgrInfo * outfn, StringInfo buf)
+{
+	appendStringInfoChar(buf, '{');
+	for (size_t i = 0; i < slot->len; i++)
+	{
+		if (i > 0)
+			appendStringInfoChar(buf, ',');
+
+		if (slot->ndim > 1)
+		{
+			ch_binary_array_t *child = (ch_binary_array_t *) DatumGetPointer(slot->datums[i]);
+
+			emit_nested_array_text(child, outfn, buf);
+		}
+		else if (slot->nulls[i])
+			appendStringInfoString(buf, "NULL");
+		else
+		{
+			char	   *s = OutputFunctionCall(outfn, slot->datums[i]);
+
+			appendStringInfoChar(buf, '"');
+			for (char *p = s; *p; p++)
+			{
+				if (*p == '"' || *p == '\\')
+					appendStringInfoChar(buf, '\\');
+				appendStringInfoChar(buf, *p);
+			}
+			appendStringInfoChar(buf, '"');
+			pfree(s);
+		}
+	}
+	appendStringInfoChar(buf, '}');
+}
+
 static Datum
 convert_array(ch_convert_state * state, Datum val)
 {
 	ch_binary_array_t *slot = (ch_binary_array_t *) DatumGetPointer(val);
 
 	if (slot->len == 0)
-		val = PointerGetDatum(construct_empty_array(state->intype));
-	else
+		val = PointerGetDatum(construct_empty_array(slot->item_type));
+	else if (slot->ndim == 1)
 	{
 		void	   *arrout = construct_array(slot->datums, slot->len, slot->item_type,
 											 state->typlen, state->typbyval, state->typalign);
 
 		val = PointerGetDatum(arrout);
 	}
+	else
+	{
+		int			dims[MAXDIM];
+		int			lbs[MAXDIM];
+		size_t		total = 1;
+		size_t		idx = 0;
+		Datum	   *flat;
+		bool	   *flatnulls;
+		ch_binary_array_t *probe = slot;
+
+		if (slot->ndim > MAXDIM)
+			ereport(ERROR,
+					(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+					 errmsg("pg_clickhouse: nested array depth %d exceeds maximum %d",
+							slot->ndim, MAXDIM)));
+
+		for (int d = 0; d < slot->ndim; d++)
+		{
+			dims[d] = (int) probe->len;
+			lbs[d] = 1;
+			total *= probe->len;
+			if (probe->ndim > 1 && probe->len > 0)
+				probe = (ch_binary_array_t *) DatumGetPointer(probe->datums[0]);
+		}
+
+		if (total == 0)
+			val = PointerGetDatum(construct_empty_array(slot->item_type));
+		else
+		{
+			flat = palloc(sizeof(Datum) * total);
+			flatnulls = palloc0(sizeof(bool) * total);
+
+			if (flatten_nested_array(slot, dims, 0, flat, flatnulls, &idx))
+				val = PointerGetDatum(construct_md_array(flat, flatnulls, slot->ndim,
+														 dims, lbs, slot->item_type,
+														 state->typlen, state->typbyval,
+														 state->typalign));
+			else
+			{
+				/*
+				 * Jagged shape: format as text and route through array_in so
+				 * binary surfaces the same malformed-literal error as http.
+				 */
+				StringInfoData buf;
+				FmgrInfo	outfn;
+				Oid			out_func;
+				Oid			in_func;
+				Oid			ioparam;
+				bool		varlena;
+
+				pfree(flat);
+				pfree(flatnulls);
+
+				getTypeOutputInfo(slot->item_type, &out_func, &varlena);
+				fmgr_info(out_func, &outfn);
+
+				initStringInfo(&buf);
+				emit_nested_array_text(slot, &outfn, &buf);
+
+				getTypeInputInfo(state->intype, &in_func, &ioparam);
+				val = OidInputFunctionCall(in_func, buf.data, ioparam, -1);
+
+				pfree(buf.data);
+			}
+		}
+	}
 
 	return convert_generic(state, val);
 }
@@ -501,6 +641,7 @@ ch_binary_do_output_conversion(ch_binary_insert_state * insert_state,
 
 				arr = palloc(sizeof(ch_binary_array_t));
 				arr->len = ArrayGetNItems(AARR_NDIM(v), AARR_DIMS(v));
+				arr->ndim = 1;
 				arr->datums = palloc(sizeof(Datum) * arr->len);
 				arr->nulls = palloc(sizeof(bool) * arr->len);
 				arr->item_type = cstate->innertype;

src/include/binary.hh

@@ -41,12 +41,21 @@ extern "C"
 		Oid		   *types;
 	}			ch_binary_tuple_t;
 
+	/*
+	 * Holds an array read from ClickHouse. For nested arrays
+	 * (Array(Array(...))) ndim > 1 and datums[i] points to a child
+	 * ch_binary_array_t with ndim-1. item_type is the leaf scalar type,
+	 * array_type is the postgres array type (same at every level since
+	 * postgres uses one array type per element type regardless of
+	 * dimensionality).
+	 */
 	typedef struct
 	{
 		Datum	   *datums;
 		bool	   *nulls;
 		size_t		len;
-		Oid			item_type;	/* used on selects */
+		int			ndim;		/* nesting depth, ≥1 (used on selects) */
+		Oid			item_type;	/* leaf element type (used on selects) */
 		Oid			array_type; /* used on selects */
 	}			ch_binary_array_t;
 

test/expected/binary.out

@@ -80,6 +80,41 @@ SELECT clickhouse_raw_query('INSERT INTO binary_test.arrays SELECT
 
 (1 row)
 
+-- nested arrays
+SELECT clickhouse_raw_query('CREATE TABLE binary_test.nested_arrays (
+    c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String))
+) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1);
+');
+ clickhouse_raw_query 
+----------------------
+ 
+(1 row)
+
+SELECT clickhouse_raw_query('INSERT INTO binary_test.nested_arrays VALUES
+    (1, [[1,2],[3,4]], [[''a'',''b''],[''c'',''d'']]),
+    (2, [[5,6],[7,8]], [[''e'',''f''],[''g'',''h'']]);
+');
+ clickhouse_raw_query 
+----------------------
+ 
+(1 row)
+
+-- ragged nested arrays must error: postgres requires hyper-rectangles
+SELECT clickhouse_raw_query('CREATE TABLE binary_test.ragged_arrays (
+    c1 Int8, c2 Array(Array(Int32))
+) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1);
+');
+ clickhouse_raw_query 
+----------------------
+ 
+(1 row)
+
+SELECT clickhouse_raw_query('INSERT INTO binary_test.ragged_arrays VALUES (1, [[1,2,3],[4]]);');
+ clickhouse_raw_query 
+----------------------
+ 
+(1 row)
+
 SELECT clickhouse_raw_query('CREATE TABLE binary_test.tuples (
     c1 Int8,
     c2 Tuple(Int, String, Float32),
@@ -152,6 +187,15 @@ CREATE FOREIGN TABLE farrays2 (
 	c1 int8[],
     c2 text[]
 ) SERVER binary_loopback OPTIONS (table_name 'arrays');
+CREATE FOREIGN TABLE fnested_arrays (
+    c1 int2,
+    c2 int[],
+    c3 text[]
+) SERVER binary_loopback OPTIONS (table_name 'nested_arrays');
+CREATE FOREIGN TABLE fragged_arrays (
+    c1 int2,
+    c2 int[]
+) SERVER binary_loopback OPTIONS (table_name 'ragged_arrays');
 CREATE TYPE tupformat AS (a int, b text, c float4);
 CREATE FOREIGN TABLE ftuples (
     c1 int,
@@ -265,6 +309,17 @@ SELECT * FROM farrays ORDER BY c1;
 
 SELECT * FROM farrays2 ORDER BY c1;
 ERROR:  pg_clickhouse: could not cast value from integer[] to bigint[]
+-- nested arrays
+SELECT * FROM fnested_arrays ORDER BY c1;
+ c1 |      c2       |      c3       
+----+---------------+---------------
+  1 | {{1,2},{3,4}} | {{a,b},{c,d}}
+  2 | {{5,6},{7,8}} | {{e,f},{g,h}}
+(2 rows)
+
+SELECT * FROM fragged_arrays ORDER BY c1;
+ERROR:  malformed array literal: "{{"1","2","3"},{"4"}}"
+DETAIL:  Multidimensional arrays must have sub-arrays with matching dimensions.
 -- tuples
 SELECT * FROM ftuples ORDER BY c1;
  c1 |    c2    | c3 
@@ -305,10 +360,12 @@ SELECT clickhouse_raw_query('DROP DATABASE binary_test');
 (1 row)
 
 DROP SERVER binary_loopback CASCADE;
-NOTICE:  drop cascades to 6 other objects
+NOTICE:  drop cascades to 8 other objects
 DETAIL:  drop cascades to foreign table fints
 drop cascades to foreign table ftypes
 drop cascades to foreign table farrays
 drop cascades to foreign table farrays2
+drop cascades to foreign table fnested_arrays
+drop cascades to foreign table fragged_arrays
 drop cascades to foreign table ftuples
 drop cascades to foreign table fbytes