Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/bin/bash
set -e

echo "Running schema evolution (complex types) test..."

SCENARIO_DIR="{{SCENARIO_DIR}}"
INPUT_PATH="${SCENARIO_DIR}/../insert-scan/input.parquet"

# Setup
{{ICE_CLI}} --config {{CLI_CONFIG}} create-namespace ${NAMESPACE_NAME}
{{ICE_CLI}} --config {{CLI_CONFIG}} insert --create-table ${TABLE_NAME} "file://${INPUT_PATH}"
echo "OK Created table ${TABLE_NAME}"

# Add list, map, struct columns in one alter-table call
{{ICE_CLI}} --config {{CLI_CONFIG}} alter-table ${TABLE_NAME} \
'[{"op":"add_column","name":"tags","type":"list<string>"},{"op":"add_column","name":"metadata","type":"map<string,long>"},{"op":"add_column","name":"address","type":"struct<street:string,zip:int>"}]'
echo "OK Altered table: added list, map, struct columns"

# Verify all three columns appear in the schema
{{ICE_CLI}} --config {{CLI_CONFIG}} describe -s ${TABLE_NAME} > /tmp/schema_ev_complex.txt
for col in tags metadata address; do
if ! grep -q "$col" /tmp/schema_ev_complex.txt; then
echo "FAIL describe -s missing expected column '${col}'"
cat /tmp/schema_ev_complex.txt
exit 1
fi
done
echo "OK Schema verified: tags, metadata, address present"

# Cleanup
{{ICE_CLI}} --config {{CLI_CONFIG}} delete-table ${TABLE_NAME}
{{ICE_CLI}} --config {{CLI_CONFIG}} delete-namespace ${NAMESPACE_NAME}
echo "OK Cleanup done"

echo "Schema evolution (complex types) test completed successfully"
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
name: "Schema evolution - add_column with complex types (list, map, struct)"
description: "Tests alter-table add_column for list<string>, map<string,long>, and struct<street:string,zip:int>"

catalogConfig:
warehouse: "s3://test-bucket/warehouse"

env:
NAMESPACE_NAME: "test_schema_ev_complex"
TABLE_NAME: "test_schema_ev_complex.t1"
6 changes: 6 additions & 0 deletions ice/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ ice alter-table flowers.iris '[{"op":"add_column","name":"extra","type":"string"
# add a NOT NULL column (`required: true`;)
ice alter-table flowers.iris '[{"op":"add_column","name":"extra","type":"string","required":true}]'

# add a list column
ice alter-table flowers.iris '[{"op":"add_column","name":"tags","type":"list<string>"}]'

# add a map column
ice alter-table flowers.iris '[{"op":"add_column","name":"metadata","type":"map<string,long>"}]'

# verify the schema change
ice describe -s flowers.iris
```
Expand Down
7 changes: 5 additions & 2 deletions ice/src/main/java/com/altinity/ice/cli/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -405,10 +405,13 @@ void alterTable(
e.g. [{"op":"drop_column","name":"foo"}]

Supported operations:
- add_column (params: "name", "type" (https://iceberg.apache.org/spec/#primitive-types), "doc" (optional),
- add_column (params: "name", "type", "doc" (optional),
"after"/"before"/"first" (optional, at most one;
position the new column after/before a named column,
or at the start of the schema))
or at the start of the schema).
"type" supports primitives (https://iceberg.apache.org/spec/#primitive-types)
and complex types: list<T>, map<K,V>, struct<name:T,...>,
e.g. "list<string>", "map<string,long>", "struct<a:string,b:long>")
- alter_column (params: "name", "type" (https://iceberg.apache.org/spec/#primitive-types))
- rename_column (params: "name", "new_name")
- drop_column (params: "name")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*/
package com.altinity.ice.cli.internal.cmd;

import com.altinity.ice.cli.internal.util.IcebergTypeParser;
import com.altinity.ice.cli.internal.util.UserInputParser;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
Expand Down Expand Up @@ -70,7 +71,7 @@ public AddColumn(
@JsonProperty("required") @Nullable Boolean required,
@JsonProperty("initial_default") @Nullable String initialDefault) {
this.name = name;
this.type = Types.fromPrimitiveString(type);
this.type = IcebergTypeParser.parseType(type);
this.doc = doc;
this.after = after;
this.before = before;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
/*
* Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*/
package com.altinity.ice.cli.internal.util;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;

/**
* Parses type strings including complex types ({@code list<string>}, {@code map<string,long>},
* {@code struct<a:string,b:long>}) into Iceberg {@link Type} instances.
*
* <p>Placeholder field IDs are assigned during parsing. Iceberg's {@code UpdateSchema.addColumn}
* reassigns fresh IDs from the table's {@code lastColumnId} on commit, so placeholder values are
* safe.
*/
public final class IcebergTypeParser {

private IcebergTypeParser() {}

public static Type parseType(String typeString) {
return parseType(typeString, new AtomicInteger(0));
}

private static Type parseType(String typeString, AtomicInteger nextId) {
String s = typeString.strip();
if (s.isEmpty()) {
throw new IllegalArgumentException("Type string must not be empty");
}

String lower = s.toLowerCase();

if (lower.startsWith("list<")) {
return parseList(s, nextId);
}
if (lower.startsWith("map<")) {
return parseMap(s, nextId);
}
if (lower.startsWith("struct<")) {
return parseStruct(s, nextId);
}

return Types.fromPrimitiveString(s);
}

private static Types.ListType parseList(String s, AtomicInteger nextId) {
String inner = unwrapAngleBrackets(s, "list");
int elementId = nextId.incrementAndGet();
Type elementType = parseType(inner, nextId);
return Types.ListType.ofOptional(elementId, elementType);
}

private static Types.MapType parseMap(String s, AtomicInteger nextId) {
String inner = unwrapAngleBrackets(s, "map");
List<String> parts = splitAtTopLevelComma(inner);
if (parts.size() != 2) {
throw new IllegalArgumentException(
"map type requires exactly 2 type parameters (key and value), got: " + s);
}
int keyId = nextId.incrementAndGet();
int valueId = nextId.incrementAndGet();
Type keyType = parseType(parts.get(0), nextId);
Type valueType = parseType(parts.get(1), nextId);
return Types.MapType.ofOptional(keyId, valueId, keyType, valueType);
}

private static Types.StructType parseStruct(String s, AtomicInteger nextId) {
String inner = unwrapAngleBrackets(s, "struct");
List<String> fieldDefs = splitAtTopLevelComma(inner);
List<Types.NestedField> fields = new ArrayList<>();
for (String fieldDef : fieldDefs) {
String trimmed = fieldDef.strip();
int colonIdx = findFieldNameSeparator(trimmed);
if (colonIdx < 0) {
throw new IllegalArgumentException(
"struct field must be in the form 'name:type', got: " + trimmed);
}
String fieldName = trimmed.substring(0, colonIdx).strip();
String fieldTypeStr = trimmed.substring(colonIdx + 1).strip();
if (fieldName.isEmpty()) {
throw new IllegalArgumentException("struct field name must not be empty in: " + trimmed);
}
if (fieldTypeStr.isEmpty()) {
throw new IllegalArgumentException("struct field type must not be empty in: " + trimmed);
}
int fieldId = nextId.incrementAndGet();
Type fieldType = parseType(fieldTypeStr, nextId);
fields.add(Types.NestedField.optional(fieldId, fieldName, fieldType));
}
return Types.StructType.of(fields);
}

/** Finds the first colon that separates name from type, skipping colons inside nested types. */
private static int findFieldNameSeparator(String s) {
int depth = 0;
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '<' || c == '(') {
depth++;
} else if (c == '>' || c == ')') {
depth--;
} else if (c == ':' && depth == 0) {
return i;
}
}
return -1;
}

/**
* Strips the prefix and the surrounding angle brackets, e.g. "list<string>" -> "string".
* Validates that brackets are present and balanced.
*/
private static String unwrapAngleBrackets(String s, String prefix) {
String trimmed = s.strip();
int prefixLen = prefix.length();
if (trimmed.length() < prefixLen + 2
|| trimmed.charAt(prefixLen) != '<'
|| trimmed.charAt(trimmed.length() - 1) != '>') {
throw new IllegalArgumentException("Expected " + prefix + "<...> syntax, got: " + trimmed);
}
String inner = trimmed.substring(prefixLen + 1, trimmed.length() - 1).strip();
if (inner.isEmpty()) {
throw new IllegalArgumentException(prefix + "<> must not be empty");
}
return inner;
}

/** Splits on commas that are not nested inside {@code <...>} or {@code (...)}. */
private static List<String> splitAtTopLevelComma(String s) {
List<String> parts = new ArrayList<>();
int depth = 0;
int start = 0;
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
if (c == '<' || c == '(') {
depth++;
} else if (c == '>' || c == ')') {
depth--;
} else if (c == ',' && depth == 0) {
parts.add(s.substring(start, i));
start = i + 1;
}
}
parts.add(s.substring(start));
return parts;
}
}
Loading
Loading