From d7658541839a6b32cc56f33c0d634c423f3b20aa Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Fri, 3 Jul 2026 17:07:26 +0200 Subject: [PATCH 01/13] [CALCITE-7639] Support bitwise right shift (>>) operator and RIGHTSHIFT function Mirrors the left-shift work in CALCITE-7109 to close a gap in the umbrella issue CALCITE-5087. Adds: * `>>` (SqlStdOperatorTable.BIT_RIGHT_SHIFT), a signed/arithmetic right shift (Java `>>`), symmetric to `<<` (precedence 32, left-assoc, ReturnTypes.ARG0_NULLABLE, InferTypes.FIRST_KNOWN, and the same INTEGER / BINARY / UNSIGNED_NUMERIC operand families). * `RIGHTSHIFT(x, n)` scalar function, mirroring `LEFTSHIFT`. * SqlFunctions.rightShift(...) runtime overloads (int, long, byte[], ByteString, and the joou unsigned types), plus BuiltInMethod.RIGHT_SHIFT and the RexImpTable registrations for both operator and function. Unlike `<<`, a greedy `>>` token cannot be added: the lexer would also match the two `>` that close nested angle-bracket types (e.g. MAP>), breaking type parsing. `>>` is therefore recognized in expression context as two adjacent `>` tokens via LOOKAHEAD(2) in BinaryRowOperator, leaving type parsing unchanged. Because there is no dedicated token, the SQL advisor advertises `>` rather than `>>`, so SqlAdvisorTest is not modified. Scope is limited to `>>` (arithmetic). The logical/fill-zero `>>>` (RIGHT_SHIFT_FILL_ZERO) remains a possible follow-up. Tests: SqlOperatorTest (operator + function forms), SqlFunctionsTest, operator.iq, and the operator-precedence dump in SqlValidatorTest; docs in site/_docs/reference.md. --- core/src/main/codegen/templates/Parser.jj | 6 + .../adapter/enumerable/RexImpTable.java | 13 ++ .../apache/calcite/runtime/SqlFunctions.java | 142 ++++++++++++++ .../calcite/sql/fun/SqlStdOperatorTable.java | 29 +++ .../apache/calcite/util/BuiltInMethod.java | 1 + .../apache/calcite/test/SqlFunctionsTest.java | 28 +++ .../apache/calcite/test/SqlValidatorTest.java | 1 + core/src/test/resources/sql/operator.iq | 31 ++++ site/_docs/reference.md | 1 + .../apache/calcite/test/SqlOperatorTest.java | 174 ++++++++++++++++++ 10 files changed, 426 insertions(+) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index c5d392e2fb0c..48c2ecceb806 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -8470,6 +8470,12 @@ SqlBinaryOperator BinaryRowOperator() : // is handled as a special case { return SqlStdOperatorTable.EQUALS; } | { return SqlStdOperatorTable.BIT_LEFT_SHIFT; } + // The right shift operator ">>" is matched as two adjacent ">" tokens rather + // than a single ">>" token. A greedy ">>" token would be produced by the lexer + // even when the two ">" characters close nested angle-bracket types (e.g. + // MAP>), breaking type parsing. Matching two ">" tokens here + // keeps right shift confined to expression context. +| LOOKAHEAD(2) { return SqlStdOperatorTable.BIT_RIGHT_SHIFT; } | { return SqlStdOperatorTable.GREATER_THAN; } | { return SqlStdOperatorTable.LESS_THAN; } | { return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; } diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index acbd66cb1cc7..b7d0456fac64 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -377,6 +377,7 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_AND; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_LEFT_SHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_OR; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_RIGHT_SHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.BIT_XOR; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.CARDINALITY; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.CAST; @@ -512,6 +513,7 @@ import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REGR_COUNT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REINTERPRET; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.REPLACE; +import static org.apache.calcite.sql.fun.SqlStdOperatorTable.RIGHTSHIFT; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROUND; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROW; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ROW_NUMBER; @@ -920,6 +922,17 @@ void populate1() { // (e.g., x << y) defineMethod(BIT_LEFT_SHIFT, BuiltInMethod.LEFT_SHIFT.method, NullPolicy.STRICT); + // Right shift operations: shift bits to the right by specified amount. + // Supports integer, unsigned integer, and binary data types. + // Shift amount is normalized using modulo arithmetic based on data type bit width. + + // RIGHTSHIFT: Function call syntax for bitwise right shift operation (e.g., RIGHTSHIFT(x, y)) + defineMethod(RIGHTSHIFT, BuiltInMethod.RIGHT_SHIFT.method, NullPolicy.STRICT); + + // BIT_RIGHT_SHIFT: Operator syntax for bitwise right shift in SQL expressions + // (e.g., x >> y) + defineMethod(BIT_RIGHT_SHIFT, BuiltInMethod.RIGHT_SHIFT.method, NullPolicy.STRICT); + define(SAFE_ADD, new SafeArithmeticImplementor(BuiltInMethod.SAFE_ADD.method)); define(SAFE_DIVIDE, diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 789fb9d6c794..726548fa1438 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3914,6 +3914,148 @@ public static ULong leftShift(ULong x, int y) { return ULong.valueOf(val); } + /** + * Performs PostgresSQL-style bitwise shift on a 32-bit integer. + * + * @param x the integer value to shift + * @param y the shift amount (positive: right shift, negative: left shift) + * @return the shifted integer + */ + public static int rightShift(int x, int y) { + int shift = ((y % 32) + 32) % 32; // normalize to 0~31 + return y >= 0 ? x >> shift : x << shift; // arithmetic right shift + } + + + // ----------------- long ----------------- + /** + * Performs PostgresSQL-style bitwise shift on a 64-bit long value. + * + * @param x the long value to shift + * @param y the shift amount + * @return the shifted long value + */ + public static long rightShift(long x, int y) { + int shift = ((y % 64) + 64) % 64; // normalize to 0~63 + return y >= 0 ? x >> shift : x << shift; + } + + /** + * Performs PostgresSQL-style bitwise shift on an int value with a long shift amount. + * + * @param x the int value to shift + * @param y the long shift amount + * @return the shifted value as long + */ + public static long rightShift(int x, long y) { + int shift = (int) (((y % 32) + 32) % 32); // normalize to 0~31 + return y >= 0 ? (long) x >> shift : (long) x << shift; + } + + /** + * Performs PostgresSQL-style bitwise shift on a byte array. + * Positive shift: right shift. + * Negative shift: treated as positive shift with modulo arithmetic. + * + * @param bytes the input byte array + * @param y the shift amount in bits + * @return the shifted byte array + */ + public static byte[] rightShift(byte[] bytes, int y) { + if (bytes.length == 0) { + return new byte[0]; + } + + int bitLen = bytes.length * 8; + + // PostgreSQL behavior: always treat as right shift with modulo arithmetic + // Negative y becomes equivalent positive shift + int shift = ((y % bitLen) + bitLen) % bitLen; + + if (shift == 0) { + return bytes.clone(); + } + + byte[] result = new byte[bytes.length]; + + // Always perform right shift (even for originally negative y) + int byteShift = shift / 8; + int bitShift = shift % 8; + + for (int i = 0; i < bytes.length; i++) { + int srcIndex = i + byteShift; + int val = 0; + + // Get the main byte + if (srcIndex < bytes.length) { + val = (bytes[srcIndex] & 0xFF) >>> bitShift; + } + + // Get carry bits from next byte + if (srcIndex + 1 < bytes.length && bitShift != 0) { + val |= (bytes[srcIndex + 1] & 0xFF) << (8 - bitShift); + } + + result[i] = (byte) val; + } + return result; + } + + /** + * Performs PostgresSQL-style bitwise shift on ByteString. + * + * @param bytes the ByteString to shift + * @param y the shift amount in bits + * @return shifted ByteString + */ + public static ByteString rightShift(ByteString bytes, int y) { + return new ByteString(rightShift(bytes.getBytes(), y)); + } + + /** + * Performs PostgresSQL-style bitwise shift on UByte. + * Overflow bits are masked to 8 bits. + */ + public static UByte rightShift(UByte x, int y) { + int shift = ((y % 8) + 8) % 8; + int val = x.byteValue() & 0xFF; + val = (y >= 0) ? (val >> shift) & 0xFF : (val << shift) & 0xFF; + return UByte.valueOf((byte) val); + } + + /** + * Performs PostgresSQL-style bitwise shift on UShort. + * Overflow bits are masked to 16 bits. + */ + public static UShort rightShift(UShort x, int y) { + int shift = ((y % 16) + 16) % 16; + int val = x.shortValue() & 0xFFFF; + val = (y >= 0) ? (val >> shift) & 0xFFFF : (val << shift) & 0xFFFF; + return UShort.valueOf((short) val); + } + + /** + * Performs PostgresSQL-style bitwise shift on UInteger. + * Overflow bits are masked to 32 bits. + */ + public static UInteger rightShift(UInteger x, int y) { + int shift = ((y % 32) + 32) % 32; + long val = x.longValue() & 0xFFFFFFFFL; + val = (y >= 0) ? (val >> shift) & 0xFFFFFFFFL : (val << shift) & 0xFFFFFFFFL; + return UInteger.valueOf(val); + } + + /** + * Performs PostgresSQL-style bitwise shift on ULong. + * Overflow bits are masked to 64 bits (long shifts naturally truncate). + */ + public static ULong rightShift(ULong x, int y) { + int shift = ((y % 64) + 64) % 64; + long val = x.longValue(); + val = (y >= 0) ? val >> shift : val << shift; + return ULong.valueOf(val); + } + // EXP /** SQL EXP operator applied to double values. */ diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index b1f0c0d5044d..f94b7052ca03 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -1396,6 +1396,35 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + /** + * {@code >>} (right shift) operator. + */ + public static final SqlBinaryOperator BIT_RIGHT_SHIFT = + new SqlBinaryOperator( + ">>", + SqlKind.OTHER, + 32, // Standard shift operator precedence + true, + ReturnTypes.ARG0_NULLABLE, + InferTypes.FIRST_KNOWN, + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), + OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), + OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + + /** + * right shift function. + */ + public static final SqlFunction RIGHTSHIFT = + SqlBasicFunction.create( + "RIGHTSHIFT", + SqlKind.OTHER_FUNCTION, + ReturnTypes.ARG0_NULLABLE, + OperandTypes.or( + OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), + OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), + OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + //------------------------------------------------------------- // WINDOW Aggregate Functions //------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 295d7d662e62..52454661671e 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -706,6 +706,7 @@ public enum BuiltInMethod { BIT_XOR(SqlFunctions.class, "bitXor", long.class, long.class), BIT_NOT(SqlFunctions.class, "bitNot", long.class), LEFT_SHIFT(SqlFunctions.class, "leftShift", int.class, int.class), + RIGHT_SHIFT(SqlFunctions.class, "rightShift", int.class, int.class), MODIFIABLE_TABLE_GET_MODIFIABLE_COLLECTION(ModifiableTable.class, "getModifiableCollection"), SCANNABLE_TABLE_SCAN(ScannableTable.class, "scan", DataContext.class), diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 959e2fabc258..3c8c5ff27dc6 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -2068,6 +2068,34 @@ private long sqlTimestamp(String str) { SqlFunctions.leftShift(new byte[]{(byte) 0x40, (byte) 0x00}, 1)); } + @Test void testRightShift() { + // Test 1-byte array + byte[] data1 = {(byte) 0xF0}; // 11110000 + for (int shift = -10; shift <= 10; shift++) { + byte[] result = SqlFunctions.rightShift(data1.clone(), shift); + // Just verify it doesn't crash and returns correct length + assertEquals(1, result.length); + } + + // Test 2-byte array + byte[] data2 = {(byte) 0x12, (byte) 0x34}; + for (int shift = -18; shift <= 18; shift++) { + byte[] result = SqlFunctions.rightShift(data2.clone(), shift); + assertEquals(2, result.length); + } + + // Verify specific known cases + assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xCD}, + SqlFunctions.rightShift(new byte[]{(byte) 0xAB, (byte) 0xCD}, 0)); + + assertArrayEquals(new byte[]{(byte) 0x40, (byte) 0x00}, + SqlFunctions.rightShift(new byte[]{(byte) 0x80, (byte) 0x00}, 1)); + + // Scalar arithmetic (sign-preserving) right shift + assertEquals(2, SqlFunctions.rightShift(8, 2)); + assertEquals(-5, SqlFunctions.rightShift(-20, 2)); + } + @Test void testCombineQueryResults() { // Test combining two equal-length lists List list1 = Arrays.asList(1, 2, 3); diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index 0fb4fa4055b6..24a6e875e182 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -11119,6 +11119,7 @@ private static int prec(SqlOperator op) { + "> SOME left\n" + ">= ALL left\n" + ">= SOME left\n" + + ">> left\n" + "BETWEEN ASYMMETRIC -\n" + "BETWEEN SYMMETRIC -\n" + "IN left\n" diff --git a/core/src/test/resources/sql/operator.iq b/core/src/test/resources/sql/operator.iq index 33731a08c4d2..41a470e5f934 100644 --- a/core/src/test/resources/sql/operator.iq +++ b/core/src/test/resources/sql/operator.iq @@ -121,6 +121,21 @@ WHERE comm IS NOT NULL LIMIT 4; !ok +# [CALCITE-7639] Add support for >> operator in Calcite +SELECT CAST(comm AS INTEGER) >> 2 AS foo FROM "scott".emp +WHERE comm IS NOT NULL LIMIT 4; ++-----+ +| FOO | ++-----+ +| 75 | +| 0 | +| 125 | +| 350 | ++-----+ +(4 rows) + +!ok + # [CALCITE-5531] COALESCE throws ClassCastException SELECT COALESCE(DATE '2021-07-08', DATE '2020-01-01') as d; +------------+ @@ -811,4 +826,20 @@ SELECT !ok +-- Bitwise RIGHT SHIFT operator `>>` +SELECT + 1 >> 0 AS shift1, + 2 >> 1 AS shift2, + 8 >> 2 AS shift3, + 32 >> 3 AS shift4, + CAST(16 AS SMALLINT) >> CAST(3 AS INTEGER) AS cast_shift; ++--------+--------+--------+--------+------------+ +| SHIFT1 | SHIFT2 | SHIFT3 | SHIFT4 | CAST_SHIFT | ++--------+--------+--------+--------+------------+ +| 1 | 1 | 2 | 4 | 2 | ++--------+--------+--------+--------+------------+ +(1 row) + +!ok + # End operator.iq diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 064d6fb95e40..40c4de1192d5 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2974,6 +2974,7 @@ In the following: | * | BITOR(value1, value2) | Returns the bitwise OR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITXOR(value1, value2) | Returns the bitwise XOR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). Negative shift amounts are converted to equivalent positive shifts through this modulo operation. For example, `LEFTSHIFT(1, -2)` returns `1073741824` (equivalent to `1 << 30`), and `LEFTSHIFT(8, -1)` returns `0` due to overflow. +| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). Negative shift amounts are converted to equivalent positive shifts through this modulo operation. For example, `RIGHTSHIFT(1024, 2)` returns `256` (equivalent to `1024 >> 2`), and `RIGHTSHIFT(-20, 2)` returns `-5`. | * | BITNOT(value) | Returns the bitwise NOT of *value*. *value* must be either an integer type or a binary value. | f | BITAND_AGG(value) | Equivalent to `BIT_AND(value)` | f | BITOR_AGG(value) | Equivalent to `BIT_OR(value)` diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 2119f79df20e..e0560d35fb80 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -17158,6 +17158,180 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkNull("LEFTSHIFT(CAST(NULL AS INTEGER UNSIGNED), 2)"); } + /** + * Test cases for + * [CALCITE-7639] + * Support bitwise right shift (>>) operator and RIGHTSHIFT function. + */ + @Test void testRightShiftScalarFunc() { + final SqlOperatorFixture f = fixture(); + f.setFor(SqlStdOperatorTable.BIT_RIGHT_SHIFT, VmName.EXPAND); + + // === Basic functionality === + f.checkScalar("8 >> 2", "2", "INTEGER NOT NULL"); + f.checkScalar("1024 >> 10", "1", "INTEGER NOT NULL"); + f.checkScalar("0 >> 5", "0", "INTEGER NOT NULL"); + + // === Type coercion and signed (arithmetic) behavior === + f.checkScalar("CAST(16 AS INTEGER) >> CAST(3 AS BIGINT)", "2", "INTEGER NOT NULL"); + f.checkScalar("-20 >> 2", "-5", "INTEGER NOT NULL"); + f.checkScalar("-40 >> 3", "-5", "INTEGER NOT NULL"); + f.checkScalar("CAST(-20 AS TINYINT) >> CAST(2 AS TINYINT)", "-5", "TINYINT NOT NULL"); + + // === Verify return type matches first argument type === + f.checkType("CAST(8 AS TINYINT) >> CAST(2 AS TINYINT)", "TINYINT NOT NULL"); + f.checkType("CAST(8 AS SMALLINT) >> CAST(2 AS SMALLINT)", "SMALLINT NOT NULL"); + f.checkType("CAST(8 AS INTEGER) >> CAST(2 AS INTEGER)", "INTEGER NOT NULL"); + f.checkType("CAST(8 AS BIGINT) >> CAST(2 AS BIGINT)", "BIGINT NOT NULL"); + + // === BigInt shifts with explicit BIGINT inputs (arithmetic/sign-preserving) === + f.checkScalar("CAST(4611686018427387904 AS BIGINT) >> 62", "1", "BIGINT NOT NULL"); // 2^62 + f.checkScalar("CAST(9223372036854775807 AS BIGINT) >> 1", + BigInteger.valueOf(Long.MAX_VALUE).shiftRight(1).toString(), "BIGINT NOT NULL"); + f.checkScalar("CAST(-1 AS BIGINT) >> 63", "-1", "BIGINT NOT NULL"); // sign bit preserved + f.checkScalar("CAST(-1 AS BIGINT) >> 1", "-1", "BIGINT NOT NULL"); + f.checkScalar("CAST(1000000000 AS BIGINT) >> 5", "31250000", "BIGINT NOT NULL"); + + // === Shift amount normalized using modulo of the bit width === + f.checkScalar("CAST(1024 AS BIGINT) >> 64", "1024", "BIGINT NOT NULL"); // 64 % 64 = 0 + f.checkScalar("CAST(1024 AS BIGINT) >> 74", "1", "BIGINT NOT NULL"); // 74 % 64 = 10 + f.checkScalar("1 >> 32", "1", "INTEGER NOT NULL"); // 32 % 32 = 0 + f.checkScalar("123 >> 60", "0", "INTEGER NOT NULL"); // 60 % 32 = 28 + + // === Unsigned types === + f.checkScalar("CAST(252 AS TINYINT UNSIGNED) >> 2", "63", "TINYINT UNSIGNED NOT NULL"); + f.checkScalar("CAST(65280 AS SMALLINT UNSIGNED) >> 8", "255", "SMALLINT UNSIGNED NOT NULL"); + f.checkScalar("CAST(4294901760 AS INTEGER UNSIGNED) >> 16", "65535", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(2147483648 AS INTEGER UNSIGNED) >> 31", "1", "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("CAST(1 AS INTEGER UNSIGNED) >> -1", "2147483648", "INTEGER UNSIGNED NOT NULL"); + + // === Negative shift counts (normalized via modulo, then shifted the other way) === + f.checkScalar("8 >> -1", "0", "INTEGER NOT NULL"); + f.checkScalar("16 >> -2", "0", "INTEGER NOT NULL"); + + // === Shift by zero and large shifts === + f.checkScalar("0 >> 32", "0", "INTEGER NOT NULL"); + f.checkScalar("0 >> 100", "0", "INTEGER NOT NULL"); + + // === Binary type tests === + f.checkScalar("CAST(X'FF' AS BINARY(1)) >> 1", "7f", "BINARY(1) NOT NULL"); + f.checkScalar("CAST(X'F0' AS BINARY(1)) >> 4", "0f", "BINARY(1) NOT NULL"); + f.checkScalar("CAST(X'08' AS BINARY(1)) >> 3", "01", "BINARY(1) NOT NULL"); + f.checkScalar("CAST(X'00' AS BINARY(1)) >> 5", "00", "BINARY(1) NOT NULL"); + + f.checkScalar("CAST(X'FFFF' AS BINARY(2)) >> 1", "ff7f", "BINARY(2) NOT NULL"); + f.checkScalar("CAST(X'1234' AS BINARY(2)) >> 4", "4103", "BINARY(2) NOT NULL"); + f.checkScalar("CAST(X'1234' AS BINARY(2)) >> 8", "3400", "BINARY(2) NOT NULL"); + + f.checkScalar("CAST(X'FF' AS BINARY(1)) >> 8", "ff", "BINARY(1) NOT NULL"); + f.checkScalar("CAST(X'FFFF' AS BINARY(2)) >> 16", "ffff", "BINARY(2) NOT NULL"); + + f.checkScalar("CAST(X'ABCD' AS BINARY(2)) >> 0", "abcd", "BINARY(2) NOT NULL"); + f.checkScalar("CAST(X'123456' AS BINARY(3)) >> 4", "416305", "BINARY(3) NOT NULL"); + f.checkScalar("CAST(X'0001' AS BINARY(2)) >> 1", "8000", "BINARY(2) NOT NULL"); + f.checkScalar("CAST(X'8000' AS BINARY(2)) >> 1", "4000", "BINARY(2) NOT NULL"); + f.checkScalar("CAST(X'F0' AS BINARY(1)) >> -4", "0f", "BINARY(1) NOT NULL"); + + // === Invalid argument types === + f.checkFails("^1.2 >> 2^", + "Cannot apply '>>' to arguments of type ' >> '\\. Supported " + + "form\\(s\\): ' >> '\\n' >> '\\n' " + + ">> '", + false); + + // === Null propagation === + f.checkNull("CAST(NULL AS INTEGER) >> 5"); + f.checkNull("10 >> CAST(NULL AS INTEGER)"); + f.checkNull("CAST(NULL AS INTEGER) >> CAST(NULL AS INTEGER)"); + f.checkNull("CAST(NULL AS INTEGER UNSIGNED) >> 2"); + } + + @Test void testRightShiftFunctionCall() { + final SqlOperatorFixture f = fixture(); + f.setFor(SqlStdOperatorTable.BIT_RIGHT_SHIFT, VmName.EXPAND); + + // === Basic functionality === + f.checkScalar("RIGHTSHIFT(8, 2)", "2", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(1024, 10)", "1", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(0, 5)", "0", "INTEGER NOT NULL"); + + // === Type coercion and signed (arithmetic) behavior === + f.checkScalar("RIGHTSHIFT(CAST(16 AS INTEGER), CAST(3 AS BIGINT))", "2", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(-20, 2)", "-5", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(-40, 3)", "-5", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(-20 AS TINYINT), CAST(2 AS TINYINT))", "-5", "TINYINT NOT NULL"); + + // === Verify return type matches first argument type === + f.checkType("RIGHTSHIFT(CAST(8 AS TINYINT), CAST(2 AS TINYINT))", "TINYINT NOT NULL"); + f.checkType("RIGHTSHIFT(CAST(8 AS SMALLINT), CAST(2 AS SMALLINT))", "SMALLINT NOT NULL"); + f.checkType("RIGHTSHIFT(CAST(8 AS INTEGER), CAST(2 AS INTEGER))", "INTEGER NOT NULL"); + f.checkType("RIGHTSHIFT(CAST(8 AS BIGINT), CAST(2 AS BIGINT))", "BIGINT NOT NULL"); + + // === BigInt shifts with explicit BIGINT inputs === + f.checkScalar("RIGHTSHIFT(CAST(4611686018427387904 AS BIGINT), 62)", "1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(9223372036854775807 AS BIGINT), 1)", + BigInteger.valueOf(Long.MAX_VALUE).shiftRight(1).toString(), "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(-1 AS BIGINT), 63)", "-1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(-1 AS BIGINT), 1)", "-1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(1000000000 AS BIGINT), 5)", "31250000", "BIGINT NOT NULL"); + + // === Shift amount normalized using modulo of the bit width === + f.checkScalar("RIGHTSHIFT(CAST(1024 AS BIGINT), 64)", "1024", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(1024 AS BIGINT), 74)", "1", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(1, 32)", "1", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(123, 60)", "0", "INTEGER NOT NULL"); + + // === Unsigned types === + f.checkScalar("RIGHTSHIFT(CAST(252 AS TINYINT UNSIGNED), 2)", "63", + "TINYINT UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(65280 AS SMALLINT UNSIGNED), 8)", "255", + "SMALLINT UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(4294901760 AS INTEGER UNSIGNED), 16)", "65535", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(2147483648 AS INTEGER UNSIGNED), 31)", "1", + "INTEGER UNSIGNED NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(1 AS INTEGER UNSIGNED), -1)", "2147483648", + "INTEGER UNSIGNED NOT NULL"); + + // === Negative shifts === + f.checkScalar("RIGHTSHIFT(8, -1)", "0", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(16, -2)", "0", "INTEGER NOT NULL"); + + // === Large shifts === + f.checkScalar("RIGHTSHIFT(0, 32)", "0", "INTEGER NOT NULL"); + f.checkScalar("RIGHTSHIFT(0, 100)", "0", "INTEGER NOT NULL"); + + // === Binary types === + f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS BINARY(1)), 1)", "7f", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS BINARY(1)), 4)", "0f", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'08' AS BINARY(1)), 3)", "01", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'00' AS BINARY(1)), 5)", "00", "BINARY(1) NOT NULL"); + + f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS BINARY(2)), 1)", "ff7f", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'1234' AS BINARY(2)), 4)", "4103", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'1234' AS BINARY(2)), 8)", "3400", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS BINARY(1)), 8)", "ff", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS BINARY(2)), 16)", "ffff", "BINARY(2) NOT NULL"); + + f.checkScalar("RIGHTSHIFT(CAST(X'ABCD' AS BINARY(2)), 0)", "abcd", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'123456' AS BINARY(3)), 4)", "416305", "BINARY(3) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'0001' AS BINARY(2)), 1)", "8000", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'8000' AS BINARY(2)), 1)", "4000", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS BINARY(1)), -4)", "0f", "BINARY(1) NOT NULL"); + // === Invalid types === + f.checkFails("^RIGHTSHIFT(1.2, 2)^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + + + // === Nulls === + f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER), 5)"); + f.checkNull("RIGHTSHIFT(10, CAST(NULL AS INTEGER))"); + f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER), CAST(NULL AS INTEGER))"); + f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER UNSIGNED), 2)"); + } + /** * Test cases for * [CALCITE-7184] From 479d69189d5c71c082c9db826eb15f8eacf11b7c Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Tue, 7 Jul 2026 15:25:30 +0200 Subject: [PATCH 02/13] [CALCITE-7639] Require adjacent '>' tokens for right shift and factor out shift operand checker Address review feedback on the right shift PR: * Parser.jj: replace the LOOKAHEAD(2) on the two '>' tokens with a semantic lookahead that checks the tokens are immediately adjacent (same line, adjacent columns). Previously '>>' matched even when the two '>' were separated by whitespace, so 'a > > b' was parsed as a right shift; it is now a syntax error, falling through to the single '>' (greater-than) alternative. * SqlStdOperatorTable: factor the duplicated operand type checker shared by '<<', '>>', LEFTSHIFT and RIGHTSHIFT into a single SHIFT_OPERAND_TYPE_CHECKER constant. * SqlParserTest: add testShiftOperators covering '<<'/'>>' parsing, precedence and left-associativity, and rejection of 'a > > b'. --- core/src/main/codegen/templates/Parser.jj | 16 +++++++--- .../calcite/sql/fun/SqlStdOperatorTable.java | 32 +++++++++---------- .../calcite/sql/parser/SqlParserTest.java | 19 +++++++++++ 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 48c2ecceb806..951732cff929 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -8470,12 +8470,18 @@ SqlBinaryOperator BinaryRowOperator() : // is handled as a special case { return SqlStdOperatorTable.EQUALS; } | { return SqlStdOperatorTable.BIT_LEFT_SHIFT; } - // The right shift operator ">>" is matched as two adjacent ">" tokens rather - // than a single ">>" token. A greedy ">>" token would be produced by the lexer - // even when the two ">" characters close nested angle-bracket types (e.g. + // The right shift operator ">>" is matched as two ">" tokens rather than a + // single ">>" token. A greedy ">>" token would be produced by the lexer even + // when the two ">" characters close nested angle-bracket types (e.g. // MAP>), breaking type parsing. Matching two ">" tokens here - // keeps right shift confined to expression context. -| LOOKAHEAD(2) { return SqlStdOperatorTable.BIT_RIGHT_SHIFT; } + // keeps right shift confined to expression context. The semantic lookahead + // requires the two ">" to be immediately adjacent (no intervening whitespace), + // so "a > > b" is not treated as a right shift; otherwise we fall through to + // the single ">" (greater-than) alternative below. +| LOOKAHEAD({ getToken(1).kind == GT && getToken(2).kind == GT + && getToken(1).endLine == getToken(2).beginLine + && getToken(1).endColumn + 1 == getToken(2).beginColumn }) + { return SqlStdOperatorTable.BIT_RIGHT_SHIFT; } | { return SqlStdOperatorTable.GREATER_THAN; } | { return SqlStdOperatorTable.LESS_THAN; } | { return SqlStdOperatorTable.LESS_THAN_OR_EQUAL; } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index f94b7052ca03..3e36b8c5d169 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -64,6 +64,7 @@ import org.apache.calcite.sql.type.OperandTypes; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; @@ -1367,6 +1368,17 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { public static final SqlAggFunction BIT_XOR = new SqlBitOpAggFunction(SqlKind.BIT_XOR); + /** + * Operand type checker shared by the shift operators ({@code <<}, {@code >>}) + * and their function forms ({@code LEFTSHIFT}, {@code RIGHTSHIFT}). The first + * operand is the value being shifted (integer, binary or unsigned numeric) and + * the second is the integer shift amount. + */ + private static final SqlOperandTypeChecker SHIFT_OPERAND_TYPE_CHECKER = + OperandTypes.INTEGER_INTEGER + .or(OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER)) + .or(OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)); + /** * {@code <<} (left shift) operator. */ @@ -1378,10 +1390,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { true, ReturnTypes.ARG0_NULLABLE, InferTypes.FIRST_KNOWN, - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + SHIFT_OPERAND_TYPE_CHECKER); /** * left shift function. @@ -1391,10 +1400,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { "LEFTSHIFT", SqlKind.OTHER_FUNCTION, ReturnTypes.ARG0_NULLABLE, - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + SHIFT_OPERAND_TYPE_CHECKER); /** * {@code >>} (right shift) operator. @@ -1407,10 +1413,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { true, ReturnTypes.ARG0_NULLABLE, InferTypes.FIRST_KNOWN, - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + SHIFT_OPERAND_TYPE_CHECKER); /** * right shift function. @@ -1420,10 +1423,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { "RIGHTSHIFT", SqlKind.OTHER_FUNCTION, ReturnTypes.ARG0_NULLABLE, - OperandTypes.or( - OperandTypes.family(SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER), - OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER))); + SHIFT_OPERAND_TYPE_CHECKER); //------------------------------------------------------------- // WINDOW Aggregate Functions diff --git a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java index bb7566c9904c..039f094b005b 100644 --- a/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java +++ b/testkit/src/main/java/org/apache/calcite/sql/parser/SqlParserTest.java @@ -1254,6 +1254,25 @@ private void checkLarge(int n) { .ok("((NOT (NOT (`A` = `B`))) OR (NOT (NOT (`C` = `D`))))"); } + @Test void testShiftOperators() { + expr("1 << 2") + .ok("(1 << 2)"); + // '>>' is recognized as two adjacent '>' tokens. + expr("1 >> 2") + .ok("(1 >> 2)"); + + // '<<' and '>>' have the same precedence and are left-associative. + expr("a << b >> c") + .ok("((`A` << `B`) >> `C`)"); + expr("a >> b >> c") + .ok("((`A` >> `B`) >> `C`)"); + + // The two '>' of a right shift must be adjacent, so "a > > b" (with a space + // between the '>' characters) is not parsed as a right shift. + expr("a ^>^ > b") + .fails("(?s).*Encountered \"> >\" at line 1, column 3\\..*"); + } + @Test void testIsBooleans() { String[] inOuts = {"NULL", "TRUE", "FALSE", "UNKNOWN"}; From 3fd6f5e56d9de27f4eed005e20557e520440f1f1 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Wed, 8 Jul 2026 11:26:06 +0200 Subject: [PATCH 03/13] [CALCITE-7639] Extract SqlParserPos.adjacent helper for right-shift token check Address review feedback: factor the '>>' token-adjacency check used by the right shift operator into a reusable SqlParserPos.adjacent(SqlParserPos) method, rather than open-coding the line/column comparison in the parser. * SqlParserPos: add adjacent(SqlParserPos), which reports whether two positions are immediately adjacent (one ends exactly where the other begins, with no characters in between). * Parser.jj: add a plain pos(Token) helper and use pos(getToken(1)).adjacent(pos(getToken(2))) in the BIT_RIGHT_SHIFT lookahead. pos is a plain method (not JAVACODE) so it can be called from generated lookahead code, which does not declare throws ParseException. * SqlParserPosTest: add a unit test for SqlParserPos.adjacent. --- core/src/main/codegen/templates/Parser.jj | 21 ++++++- .../calcite/sql/parser/SqlParserPos.java | 28 +++++++++ .../calcite/sql/parser/SqlParserPosTest.java | 59 +++++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index 951732cff929..ceece694a7c6 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -279,6 +279,24 @@ public class ${parser.class} extends SqlAbstractParserImpl Span.of(table, extendList).pos(), table, extendList); } + /** + * Returns the parser position of the given token. + * + *

Declared as a plain method (rather than JAVACODE) so it can be called + * from generated lookahead code, which does not declare + * {@code throws ParseException}. + * + * @param token token whose position to return + * @return parser position spanning the given token + */ + private static SqlParserPos pos(Token token) { + return new SqlParserPos( + token.beginLine, + token.beginColumn, + token.endLine, + token.endColumn); + } + /** Adds a warning that a token such as "HOURS" was used, * whereas the SQL standard only allows "HOUR". * @@ -8479,8 +8497,7 @@ SqlBinaryOperator BinaryRowOperator() : // so "a > > b" is not treated as a right shift; otherwise we fall through to // the single ">" (greater-than) alternative below. | LOOKAHEAD({ getToken(1).kind == GT && getToken(2).kind == GT - && getToken(1).endLine == getToken(2).beginLine - && getToken(1).endColumn + 1 == getToken(2).beginColumn }) + && pos(getToken(1)).adjacent(pos(getToken(2))) }) { return SqlStdOperatorTable.BIT_RIGHT_SHIFT; } | { return SqlStdOperatorTable.GREATER_THAN; } | { return SqlStdOperatorTable.LESS_THAN; } diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java index 97fb72cac001..d3149e617856 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java @@ -264,6 +264,34 @@ public boolean startsAt(SqlParserPos pos) { && columnNumber == pos.columnNumber; } + /** + * Returns whether this position is immediately adjacent to another position; + * that is, one position ends exactly where the other begins, with no + * characters (such as whitespace) in between. + * + *

For example, the two {@code >} characters in {@code >>} are adjacent, + * whereas those in {@code > >} are not. + * + * @param pos position to compare against + * @return whether this position and {@code pos} are immediately adjacent + */ + public boolean adjacent(SqlParserPos pos) { + return endsImmediatelyBefore(pos) || pos.endsImmediatelyBefore(this); + } + + /** + * Returns whether this position ends exactly one column before another + * position begins, on the same line. + * + * @param pos position that may immediately follow this one + * @return whether this position ends exactly one column before {@code pos} + * begins, on the same line + */ + private boolean endsImmediatelyBefore(SqlParserPos pos) { + return endLineNumber == pos.lineNumber + && endColumnNumber + 1 == pos.columnNumber; + } + /** Parser position for an identifier segment that is quoted. */ private static class QuotedParserPos extends SqlParserPos { QuotedParserPos(int startLineNumber, int startColumnNumber, diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java new file mode 100644 index 000000000000..0b182e605e32 --- /dev/null +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.parser; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * Tests for {@link SqlParserPos}. + */ +public class SqlParserPosTest { + /** Tests {@link SqlParserPos#adjacent(SqlParserPos)}. */ + @Test void testAdjacent() { + // Two single-character positions in consecutive columns are adjacent, + // like the two '>' of a '>>' token. + final SqlParserPos col1 = new SqlParserPos(1, 1); + final SqlParserPos col2 = new SqlParserPos(1, 2); + assertThat(col1.adjacent(col2), is(true)); + + // Adjacency is symmetric. + assertThat(col2.adjacent(col1), is(true)); + + // A gap between the positions (e.g. whitespace, like '> >') is not + // adjacent. + final SqlParserPos col3 = new SqlParserPos(1, 3); + assertThat(col1.adjacent(col3), is(false)); + assertThat(col3.adjacent(col1), is(false)); + + // A position is not adjacent to itself. + assertThat(col1.adjacent(col1), is(false)); + + // A multi-column position is adjacent to the position that starts one + // column after it ends. + final SqlParserPos cols1To2 = new SqlParserPos(1, 1, 1, 2); + assertThat(cols1To2.adjacent(col3), is(true)); + assertThat(cols1To2.adjacent(col2), is(false)); + + // Positions on different lines are never adjacent. + final SqlParserPos line2 = new SqlParserPos(2, 1); + assertThat(new SqlParserPos(1, 5).adjacent(line2), is(false)); + assertThat(line2.adjacent(new SqlParserPos(1, 5)), is(false)); + } +} From 1bec017247ca10cdd45c32db7e3e584675725684 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Thu, 9 Jul 2026 08:07:06 +0200 Subject: [PATCH 04/13] [CALCITE-7639] Simplify shift helpers: endsImmediatelyBefore, Math.floorMod, unsigned-amount tests Address further review feedback: * SqlParserPos: replace the symmetric adjacent(SqlParserPos) with the directional endsImmediatelyBefore(SqlParserPos), which is more precise for the right-shift token check and is now used directly by the parser. * SqlFunctions: replace the hand-rolled ((y % w) + w) % w shift-amount normalization with Math.floorMod(y, w) throughout leftShift/rightShift. * SqlOperatorTest: add tests showing a BIGINT shift amount is accepted while an unsigned shift amount is rejected at validation, for both the >> operator and the RIGHTSHIFT function. --- core/src/main/codegen/templates/Parser.jj | 2 +- .../apache/calcite/runtime/SqlFunctions.java | 32 ++++++++-------- .../calcite/sql/parser/SqlParserPos.java | 20 ++-------- .../calcite/sql/parser/SqlParserPosTest.java | 38 +++++++++---------- .../apache/calcite/test/SqlOperatorTest.java | 15 ++++++++ 5 files changed, 53 insertions(+), 54 deletions(-) diff --git a/core/src/main/codegen/templates/Parser.jj b/core/src/main/codegen/templates/Parser.jj index ceece694a7c6..b9c295211c9d 100644 --- a/core/src/main/codegen/templates/Parser.jj +++ b/core/src/main/codegen/templates/Parser.jj @@ -8497,7 +8497,7 @@ SqlBinaryOperator BinaryRowOperator() : // so "a > > b" is not treated as a right shift; otherwise we fall through to // the single ">" (greater-than) alternative below. | LOOKAHEAD({ getToken(1).kind == GT && getToken(2).kind == GT - && pos(getToken(1)).adjacent(pos(getToken(2))) }) + && pos(getToken(1)).endsImmediatelyBefore(pos(getToken(2))) }) { return SqlStdOperatorTable.BIT_RIGHT_SHIFT; } | { return SqlStdOperatorTable.GREATER_THAN; } | { return SqlStdOperatorTable.LESS_THAN; } diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 726548fa1438..f5536df251dc 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3780,7 +3780,7 @@ private static ByteString binaryOperator( * @return the shifted integer */ public static int leftShift(int x, int y) { - int shift = ((y % 32) + 32) % 32; // normalize to 0~31 + int shift = Math.floorMod(y, 32); // normalize to 0~31 return y >= 0 ? x << shift : x >> shift; // arithmetic right shift } @@ -3794,7 +3794,7 @@ public static int leftShift(int x, int y) { * @return the shifted long value */ public static long leftShift(long x, int y) { - int shift = ((y % 64) + 64) % 64; // normalize to 0~63 + int shift = Math.floorMod(y, 64); // normalize to 0~63 return y >= 0 ? x << shift : x >> shift; } @@ -3806,7 +3806,7 @@ public static long leftShift(long x, int y) { * @return the shifted value as long */ public static long leftShift(int x, long y) { - int shift = (int) (((y % 32) + 32) % 32); // normalize to 0~31 + int shift = Math.floorMod(y, 32); // normalize to 0~31 return y >= 0 ? (long) x << shift : (long) x >> shift; } @@ -3828,7 +3828,7 @@ public static byte[] leftShift(byte[] bytes, int y) { // PostgreSQL behavior: always treat as left shift with modulo arithmetic // Negative y becomes equivalent positive shift - int shift = ((y % bitLen) + bitLen) % bitLen; + int shift = Math.floorMod(y, bitLen); if (shift == 0) { return bytes.clone(); @@ -3875,7 +3875,7 @@ public static ByteString leftShift(ByteString bytes, int y) { * Overflow bits are masked to 8 bits. */ public static UByte leftShift(UByte x, int y) { - int shift = ((y % 8) + 8) % 8; + int shift = Math.floorMod(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val << shift) & 0xFF : (val >> shift) & 0xFF; return UByte.valueOf((byte) val); @@ -3886,7 +3886,7 @@ public static UByte leftShift(UByte x, int y) { * Overflow bits are masked to 16 bits. */ public static UShort leftShift(UShort x, int y) { - int shift = ((y % 16) + 16) % 16; + int shift = Math.floorMod(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val << shift) & 0xFFFF : (val >> shift) & 0xFFFF; return UShort.valueOf((short) val); @@ -3897,7 +3897,7 @@ public static UShort leftShift(UShort x, int y) { * Overflow bits are masked to 32 bits. */ public static UInteger leftShift(UInteger x, int y) { - int shift = ((y % 32) + 32) % 32; + int shift = Math.floorMod(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val << shift) & 0xFFFFFFFFL : (val >> shift) & 0xFFFFFFFFL; return UInteger.valueOf(val); @@ -3908,7 +3908,7 @@ public static UInteger leftShift(UInteger x, int y) { * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ public static ULong leftShift(ULong x, int y) { - int shift = ((y % 64) + 64) % 64; + int shift = Math.floorMod(y, 64); long val = x.longValue(); val = (y >= 0) ? val << shift : val >> shift; return ULong.valueOf(val); @@ -3922,7 +3922,7 @@ public static ULong leftShift(ULong x, int y) { * @return the shifted integer */ public static int rightShift(int x, int y) { - int shift = ((y % 32) + 32) % 32; // normalize to 0~31 + int shift = Math.floorMod(y, 32); // normalize to 0~31 return y >= 0 ? x >> shift : x << shift; // arithmetic right shift } @@ -3936,7 +3936,7 @@ public static int rightShift(int x, int y) { * @return the shifted long value */ public static long rightShift(long x, int y) { - int shift = ((y % 64) + 64) % 64; // normalize to 0~63 + int shift = Math.floorMod(y, 64); // normalize to 0~63 return y >= 0 ? x >> shift : x << shift; } @@ -3948,7 +3948,7 @@ public static long rightShift(long x, int y) { * @return the shifted value as long */ public static long rightShift(int x, long y) { - int shift = (int) (((y % 32) + 32) % 32); // normalize to 0~31 + int shift = Math.floorMod(y, 32); // normalize to 0~31 return y >= 0 ? (long) x >> shift : (long) x << shift; } @@ -3970,7 +3970,7 @@ public static byte[] rightShift(byte[] bytes, int y) { // PostgreSQL behavior: always treat as right shift with modulo arithmetic // Negative y becomes equivalent positive shift - int shift = ((y % bitLen) + bitLen) % bitLen; + int shift = Math.floorMod(y, bitLen); if (shift == 0) { return bytes.clone(); @@ -4017,7 +4017,7 @@ public static ByteString rightShift(ByteString bytes, int y) { * Overflow bits are masked to 8 bits. */ public static UByte rightShift(UByte x, int y) { - int shift = ((y % 8) + 8) % 8; + int shift = Math.floorMod(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val >> shift) & 0xFF : (val << shift) & 0xFF; return UByte.valueOf((byte) val); @@ -4028,7 +4028,7 @@ public static UByte rightShift(UByte x, int y) { * Overflow bits are masked to 16 bits. */ public static UShort rightShift(UShort x, int y) { - int shift = ((y % 16) + 16) % 16; + int shift = Math.floorMod(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val >> shift) & 0xFFFF : (val << shift) & 0xFFFF; return UShort.valueOf((short) val); @@ -4039,7 +4039,7 @@ public static UShort rightShift(UShort x, int y) { * Overflow bits are masked to 32 bits. */ public static UInteger rightShift(UInteger x, int y) { - int shift = ((y % 32) + 32) % 32; + int shift = Math.floorMod(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val >> shift) & 0xFFFFFFFFL : (val << shift) & 0xFFFFFFFFL; return UInteger.valueOf(val); @@ -4050,7 +4050,7 @@ public static UInteger rightShift(UInteger x, int y) { * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ public static ULong rightShift(ULong x, int y) { - int shift = ((y % 64) + 64) % 64; + int shift = Math.floorMod(y, 64); long val = x.longValue(); val = (y >= 0) ? val >> shift : val << shift; return ULong.valueOf(val); diff --git a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java index d3149e617856..ed74719ab332 100644 --- a/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java +++ b/core/src/main/java/org/apache/calcite/sql/parser/SqlParserPos.java @@ -264,30 +264,16 @@ public boolean startsAt(SqlParserPos pos) { && columnNumber == pos.columnNumber; } - /** - * Returns whether this position is immediately adjacent to another position; - * that is, one position ends exactly where the other begins, with no - * characters (such as whitespace) in between. - * - *

For example, the two {@code >} characters in {@code >>} are adjacent, - * whereas those in {@code > >} are not. - * - * @param pos position to compare against - * @return whether this position and {@code pos} are immediately adjacent - */ - public boolean adjacent(SqlParserPos pos) { - return endsImmediatelyBefore(pos) || pos.endsImmediatelyBefore(this); - } - /** * Returns whether this position ends exactly one column before another - * position begins, on the same line. + * position begins, on the same line, with no characters (such as whitespace) + * in between. * * @param pos position that may immediately follow this one * @return whether this position ends exactly one column before {@code pos} * begins, on the same line */ - private boolean endsImmediatelyBefore(SqlParserPos pos) { + public boolean endsImmediatelyBefore(SqlParserPos pos) { return endLineNumber == pos.lineNumber && endColumnNumber + 1 == pos.columnNumber; } diff --git a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java index 0b182e605e32..ed6d07e77c6b 100644 --- a/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java +++ b/core/src/test/java/org/apache/calcite/sql/parser/SqlParserPosTest.java @@ -25,35 +25,33 @@ * Tests for {@link SqlParserPos}. */ public class SqlParserPosTest { - /** Tests {@link SqlParserPos#adjacent(SqlParserPos)}. */ - @Test void testAdjacent() { - // Two single-character positions in consecutive columns are adjacent, - // like the two '>' of a '>>' token. + /** Tests {@link SqlParserPos#endsImmediatelyBefore(SqlParserPos)}. */ + @Test void testEndsImmediatelyBefore() { + // A single-character position ends immediately before the one in the next + // column, like the two '>' of a '>>' token. final SqlParserPos col1 = new SqlParserPos(1, 1); final SqlParserPos col2 = new SqlParserPos(1, 2); - assertThat(col1.adjacent(col2), is(true)); + assertThat(col1.endsImmediatelyBefore(col2), is(true)); - // Adjacency is symmetric. - assertThat(col2.adjacent(col1), is(true)); + // The relation is directional, not symmetric. + assertThat(col2.endsImmediatelyBefore(col1), is(false)); - // A gap between the positions (e.g. whitespace, like '> >') is not - // adjacent. + // A gap between the positions (e.g. whitespace, like '> >') does not + // qualify. final SqlParserPos col3 = new SqlParserPos(1, 3); - assertThat(col1.adjacent(col3), is(false)); - assertThat(col3.adjacent(col1), is(false)); + assertThat(col1.endsImmediatelyBefore(col3), is(false)); - // A position is not adjacent to itself. - assertThat(col1.adjacent(col1), is(false)); + // A position does not end immediately before itself. + assertThat(col1.endsImmediatelyBefore(col1), is(false)); - // A multi-column position is adjacent to the position that starts one - // column after it ends. + // A multi-column position ends immediately before the position that starts + // one column after it ends. final SqlParserPos cols1To2 = new SqlParserPos(1, 1, 1, 2); - assertThat(cols1To2.adjacent(col3), is(true)); - assertThat(cols1To2.adjacent(col2), is(false)); + assertThat(cols1To2.endsImmediatelyBefore(col3), is(true)); + assertThat(cols1To2.endsImmediatelyBefore(col2), is(false)); - // Positions on different lines are never adjacent. + // Positions on different lines never qualify. final SqlParserPos line2 = new SqlParserPos(2, 1); - assertThat(new SqlParserPos(1, 5).adjacent(line2), is(false)); - assertThat(line2.adjacent(new SqlParserPos(1, 5)), is(false)); + assertThat(new SqlParserPos(1, 5).endsImmediatelyBefore(line2), is(false)); } } diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index e0560d35fb80..aa511385da23 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -17206,6 +17206,15 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkScalar("CAST(2147483648 AS INTEGER UNSIGNED) >> 31", "1", "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(1 AS INTEGER UNSIGNED) >> -1", "2147483648", "INTEGER UNSIGNED NOT NULL"); + // A BIGINT shift amount is accepted (INTEGER family), but an unsigned shift + // amount is not: the second operand must be a signed integer type. + f.checkScalar("CAST(8 AS INTEGER) >> CAST(2 AS BIGINT)", "2", "INTEGER NOT NULL"); + f.checkFails("^8 >> CAST(2 AS INTEGER UNSIGNED)^", + "Cannot apply '>>' to arguments of type ' >> '\\. " + + "Supported form\\(s\\): ' >> '\\n' >> '\\n" + + "' >> '", + false); + // === Negative shift counts (normalized via modulo, then shifted the other way) === f.checkScalar("8 >> -1", "0", "INTEGER NOT NULL"); f.checkScalar("16 >> -2", "0", "INTEGER NOT NULL"); @@ -17323,6 +17332,12 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkFails("^RIGHTSHIFT(1.2, 2)^", "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", false); + // A BIGINT shift amount is accepted, but an unsigned shift amount is not: + // the second argument must be a signed integer type. + f.checkScalar("RIGHTSHIFT(8, CAST(2 AS BIGINT))", "2", "INTEGER NOT NULL"); + f.checkFails("^RIGHTSHIFT(8, CAST(2 AS INTEGER UNSIGNED))^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); // === Nulls === From 4add3af2f778dd9552901e2a8a87593191111124 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Thu, 9 Jul 2026 08:25:16 +0200 Subject: [PATCH 05/13] [CALCITE-7639] Use a single long shift-amount overload for the shift functions Address review feedback on the inconsistent shift-amount overloads: the int-valued shift methods had both an int and a long shift-amount variant, while the long and unsigned variants only had an int shift amount, and no (long, long) overload existed at all (so an executed BIGINT >> BIGINT would fail to resolve at code generation). Make every leftShift/rightShift overload take a long shift amount. An int shift amount widens to long automatically during enumerable code generation, so a single amount type suffices for every value type; the redundant int-valued long-amount overloads are removed. Behaviour is unchanged at the SQL level (the result type still follows the first argument). * SqlFunctions: change the shift amount of every leftShift/rightShift overload from int to long, and drop the now-redundant leftShift(int, long) / rightShift(int, long) overloads. * BuiltInMethod: point LEFT_SHIFT/RIGHT_SHIFT at the (int, long) overloads. * SqlOperatorTest: add executed BIGINT-by-BIGINT cases for both the operator and function forms of left and right shift. --- .../apache/calcite/runtime/SqlFunctions.java | 56 ++++++------------- .../apache/calcite/util/BuiltInMethod.java | 4 +- .../apache/calcite/test/SqlOperatorTest.java | 4 ++ 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index f5536df251dc..4fa76eb14375 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3779,7 +3779,7 @@ private static ByteString binaryOperator( * @param y the shift amount (positive: left shift, negative: right shift) * @return the shifted integer */ - public static int leftShift(int x, int y) { + public static int leftShift(int x, long y) { int shift = Math.floorMod(y, 32); // normalize to 0~31 return y >= 0 ? x << shift : x >> shift; // arithmetic right shift } @@ -3793,23 +3793,11 @@ public static int leftShift(int x, int y) { * @param y the shift amount * @return the shifted long value */ - public static long leftShift(long x, int y) { + public static long leftShift(long x, long y) { int shift = Math.floorMod(y, 64); // normalize to 0~63 return y >= 0 ? x << shift : x >> shift; } - /** - * Performs PostgresSQL-style bitwise shift on an int value with a long shift amount. - * - * @param x the int value to shift - * @param y the long shift amount - * @return the shifted value as long - */ - public static long leftShift(int x, long y) { - int shift = Math.floorMod(y, 32); // normalize to 0~31 - return y >= 0 ? (long) x << shift : (long) x >> shift; - } - /** * Performs PostgresSQL-style bitwise shift on a byte array. * Positive shift: left shift. @@ -3819,7 +3807,7 @@ public static long leftShift(int x, long y) { * @param y the shift amount in bits * @return the shifted byte array */ - public static byte[] leftShift(byte[] bytes, int y) { + public static byte[] leftShift(byte[] bytes, long y) { if (bytes.length == 0) { return new byte[0]; } @@ -3866,7 +3854,7 @@ public static byte[] leftShift(byte[] bytes, int y) { * @param y the shift amount in bits * @return shifted ByteString */ - public static ByteString leftShift(ByteString bytes, int y) { + public static ByteString leftShift(ByteString bytes, long y) { return new ByteString(leftShift(bytes.getBytes(), y)); } @@ -3874,7 +3862,7 @@ public static ByteString leftShift(ByteString bytes, int y) { * Performs PostgresSQL-style bitwise shift on UByte. * Overflow bits are masked to 8 bits. */ - public static UByte leftShift(UByte x, int y) { + public static UByte leftShift(UByte x, long y) { int shift = Math.floorMod(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val << shift) & 0xFF : (val >> shift) & 0xFF; @@ -3885,7 +3873,7 @@ public static UByte leftShift(UByte x, int y) { * Performs PostgresSQL-style bitwise shift on UShort. * Overflow bits are masked to 16 bits. */ - public static UShort leftShift(UShort x, int y) { + public static UShort leftShift(UShort x, long y) { int shift = Math.floorMod(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val << shift) & 0xFFFF : (val >> shift) & 0xFFFF; @@ -3896,7 +3884,7 @@ public static UShort leftShift(UShort x, int y) { * Performs PostgresSQL-style bitwise shift on UInteger. * Overflow bits are masked to 32 bits. */ - public static UInteger leftShift(UInteger x, int y) { + public static UInteger leftShift(UInteger x, long y) { int shift = Math.floorMod(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val << shift) & 0xFFFFFFFFL : (val >> shift) & 0xFFFFFFFFL; @@ -3907,7 +3895,7 @@ public static UInteger leftShift(UInteger x, int y) { * Performs PostgresSQL-style bitwise shift on ULong. * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ - public static ULong leftShift(ULong x, int y) { + public static ULong leftShift(ULong x, long y) { int shift = Math.floorMod(y, 64); long val = x.longValue(); val = (y >= 0) ? val << shift : val >> shift; @@ -3921,7 +3909,7 @@ public static ULong leftShift(ULong x, int y) { * @param y the shift amount (positive: right shift, negative: left shift) * @return the shifted integer */ - public static int rightShift(int x, int y) { + public static int rightShift(int x, long y) { int shift = Math.floorMod(y, 32); // normalize to 0~31 return y >= 0 ? x >> shift : x << shift; // arithmetic right shift } @@ -3935,23 +3923,11 @@ public static int rightShift(int x, int y) { * @param y the shift amount * @return the shifted long value */ - public static long rightShift(long x, int y) { + public static long rightShift(long x, long y) { int shift = Math.floorMod(y, 64); // normalize to 0~63 return y >= 0 ? x >> shift : x << shift; } - /** - * Performs PostgresSQL-style bitwise shift on an int value with a long shift amount. - * - * @param x the int value to shift - * @param y the long shift amount - * @return the shifted value as long - */ - public static long rightShift(int x, long y) { - int shift = Math.floorMod(y, 32); // normalize to 0~31 - return y >= 0 ? (long) x >> shift : (long) x << shift; - } - /** * Performs PostgresSQL-style bitwise shift on a byte array. * Positive shift: right shift. @@ -3961,7 +3937,7 @@ public static long rightShift(int x, long y) { * @param y the shift amount in bits * @return the shifted byte array */ - public static byte[] rightShift(byte[] bytes, int y) { + public static byte[] rightShift(byte[] bytes, long y) { if (bytes.length == 0) { return new byte[0]; } @@ -4008,7 +3984,7 @@ public static byte[] rightShift(byte[] bytes, int y) { * @param y the shift amount in bits * @return shifted ByteString */ - public static ByteString rightShift(ByteString bytes, int y) { + public static ByteString rightShift(ByteString bytes, long y) { return new ByteString(rightShift(bytes.getBytes(), y)); } @@ -4016,7 +3992,7 @@ public static ByteString rightShift(ByteString bytes, int y) { * Performs PostgresSQL-style bitwise shift on UByte. * Overflow bits are masked to 8 bits. */ - public static UByte rightShift(UByte x, int y) { + public static UByte rightShift(UByte x, long y) { int shift = Math.floorMod(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val >> shift) & 0xFF : (val << shift) & 0xFF; @@ -4027,7 +4003,7 @@ public static UByte rightShift(UByte x, int y) { * Performs PostgresSQL-style bitwise shift on UShort. * Overflow bits are masked to 16 bits. */ - public static UShort rightShift(UShort x, int y) { + public static UShort rightShift(UShort x, long y) { int shift = Math.floorMod(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val >> shift) & 0xFFFF : (val << shift) & 0xFFFF; @@ -4038,7 +4014,7 @@ public static UShort rightShift(UShort x, int y) { * Performs PostgresSQL-style bitwise shift on UInteger. * Overflow bits are masked to 32 bits. */ - public static UInteger rightShift(UInteger x, int y) { + public static UInteger rightShift(UInteger x, long y) { int shift = Math.floorMod(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val >> shift) & 0xFFFFFFFFL : (val << shift) & 0xFFFFFFFFL; @@ -4049,7 +4025,7 @@ public static UInteger rightShift(UInteger x, int y) { * Performs PostgresSQL-style bitwise shift on ULong. * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ - public static ULong rightShift(ULong x, int y) { + public static ULong rightShift(ULong x, long y) { int shift = Math.floorMod(y, 64); long val = x.longValue(); val = (y >= 0) ? val >> shift : val << shift; diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 52454661671e..a994234ddeee 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -705,8 +705,8 @@ public enum BuiltInMethod { BIT_OR(SqlFunctions.class, "bitOr", long.class, long.class), BIT_XOR(SqlFunctions.class, "bitXor", long.class, long.class), BIT_NOT(SqlFunctions.class, "bitNot", long.class), - LEFT_SHIFT(SqlFunctions.class, "leftShift", int.class, int.class), - RIGHT_SHIFT(SqlFunctions.class, "rightShift", int.class, int.class), + LEFT_SHIFT(SqlFunctions.class, "leftShift", int.class, long.class), + RIGHT_SHIFT(SqlFunctions.class, "rightShift", int.class, long.class), MODIFIABLE_TABLE_GET_MODIFIABLE_COLLECTION(ModifiableTable.class, "getModifiableCollection"), SCANNABLE_TABLE_SCAN(ScannableTable.class, "scan", DataContext.class), diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index aa511385da23..306957321b61 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -16958,6 +16958,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkType("CAST(2 AS SMALLINT) << CAST(3 AS SMALLINT)", "SMALLINT NOT NULL"); f.checkType("CAST(2 AS INTEGER) << CAST(3 AS INTEGER)", "INTEGER NOT NULL"); f.checkType("CAST(2 AS BIGINT) << CAST(3 AS BIGINT)", "BIGINT NOT NULL"); + f.checkScalar("CAST(2 AS BIGINT) << CAST(3 AS BIGINT)", "16", "BIGINT NOT NULL"); // === BigInt shifts with explicit BIGINT inputs === f.checkScalar("CAST(1 AS BIGINT) << 62", BigInteger.ONE.shiftLeft(62).toString(), @@ -17078,6 +17079,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkType("LEFTSHIFT(CAST(2 AS SMALLINT), CAST(3 AS SMALLINT))", "SMALLINT NOT NULL"); f.checkType("LEFTSHIFT(CAST(2 AS INTEGER), CAST(3 AS INTEGER))", "INTEGER NOT NULL"); f.checkType("LEFTSHIFT(CAST(2 AS BIGINT), CAST(3 AS BIGINT))", "BIGINT NOT NULL"); + f.checkScalar("LEFTSHIFT(CAST(2 AS BIGINT), CAST(3 AS BIGINT))", "16", "BIGINT NOT NULL"); // === BigInt shifts with explicit BIGINT inputs === f.checkScalar("LEFTSHIFT(CAST(1 AS BIGINT), 62)", @@ -17183,6 +17185,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkType("CAST(8 AS SMALLINT) >> CAST(2 AS SMALLINT)", "SMALLINT NOT NULL"); f.checkType("CAST(8 AS INTEGER) >> CAST(2 AS INTEGER)", "INTEGER NOT NULL"); f.checkType("CAST(8 AS BIGINT) >> CAST(2 AS BIGINT)", "BIGINT NOT NULL"); + f.checkScalar("CAST(8 AS BIGINT) >> CAST(2 AS BIGINT)", "2", "BIGINT NOT NULL"); // === BigInt shifts with explicit BIGINT inputs (arithmetic/sign-preserving) === f.checkScalar("CAST(4611686018427387904 AS BIGINT) >> 62", "1", "BIGINT NOT NULL"); // 2^62 @@ -17276,6 +17279,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkType("RIGHTSHIFT(CAST(8 AS SMALLINT), CAST(2 AS SMALLINT))", "SMALLINT NOT NULL"); f.checkType("RIGHTSHIFT(CAST(8 AS INTEGER), CAST(2 AS INTEGER))", "INTEGER NOT NULL"); f.checkType("RIGHTSHIFT(CAST(8 AS BIGINT), CAST(2 AS BIGINT))", "BIGINT NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(8 AS BIGINT), CAST(2 AS BIGINT))", "2", "BIGINT NOT NULL"); // === BigInt shifts with explicit BIGINT inputs === f.checkScalar("RIGHTSHIFT(CAST(4611686018427387904 AS BIGINT), 62)", "1", "BIGINT NOT NULL"); From 0d85f00c076e2a8eea9aaa7b998d4627a4f86acc Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Thu, 9 Jul 2026 08:46:25 +0200 Subject: [PATCH 06/13] [CALCITE-7639] Normalize shift amounts with a JDK 8 compatible positiveModulo helper An earlier commit made the shift amount a long and normalized it with Math.floorMod(y, ). Math.floorMod(long, int) was only added in JDK 9, so on JDK 8 the call resolves to Math.floorMod(long, long) (returning long) and does not compile when assigned to an int ('incompatible types: possible lossy conversion from long to int'). Introduce a private positiveModulo(long, int) helper that widens to Math.floorMod(long, long) and narrows the result (always in [0, m)) back to int, and route all shift-amount normalizations through it. This keeps the JDK 8 build working and factors the modulo into a single named helper. --- .../apache/calcite/runtime/SqlFunctions.java | 43 +++++++++++++------ 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 4fa76eb14375..ec5802e1600d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3772,6 +3772,21 @@ private static ByteString binaryOperator( return new ByteString(result); } + /** + * Returns {@code x} modulo {@code m}, normalized to the range {@code [0, m)} + * (unlike {@code %}, the result is never negative). Used to normalize a shift + * amount to the bit width of the value being shifted. + * + * @param x the value (typically a shift amount, which may be negative) + * @param m the modulus, which must be positive (typically a bit width) + * @return {@code x} modulo {@code m}, in the range {@code [0, m)} + */ + private static int positiveModulo(long x, int m) { + // Math.floorMod(long, int) is only available since JDK 9, so widen to + // Math.floorMod(long, long) and narrow the result (always in [0, m)) to int. + return (int) Math.floorMod(x, (long) m); + } + /** * Performs PostgresSQL-style bitwise shift on a 32-bit integer. * @@ -3780,7 +3795,7 @@ private static ByteString binaryOperator( * @return the shifted integer */ public static int leftShift(int x, long y) { - int shift = Math.floorMod(y, 32); // normalize to 0~31 + int shift = positiveModulo(y, 32); // normalize to 0~31 return y >= 0 ? x << shift : x >> shift; // arithmetic right shift } @@ -3794,7 +3809,7 @@ public static int leftShift(int x, long y) { * @return the shifted long value */ public static long leftShift(long x, long y) { - int shift = Math.floorMod(y, 64); // normalize to 0~63 + int shift = positiveModulo(y, 64); // normalize to 0~63 return y >= 0 ? x << shift : x >> shift; } @@ -3816,7 +3831,7 @@ public static byte[] leftShift(byte[] bytes, long y) { // PostgreSQL behavior: always treat as left shift with modulo arithmetic // Negative y becomes equivalent positive shift - int shift = Math.floorMod(y, bitLen); + int shift = positiveModulo(y, bitLen); if (shift == 0) { return bytes.clone(); @@ -3863,7 +3878,7 @@ public static ByteString leftShift(ByteString bytes, long y) { * Overflow bits are masked to 8 bits. */ public static UByte leftShift(UByte x, long y) { - int shift = Math.floorMod(y, 8); + int shift = positiveModulo(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val << shift) & 0xFF : (val >> shift) & 0xFF; return UByte.valueOf((byte) val); @@ -3874,7 +3889,7 @@ public static UByte leftShift(UByte x, long y) { * Overflow bits are masked to 16 bits. */ public static UShort leftShift(UShort x, long y) { - int shift = Math.floorMod(y, 16); + int shift = positiveModulo(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val << shift) & 0xFFFF : (val >> shift) & 0xFFFF; return UShort.valueOf((short) val); @@ -3885,7 +3900,7 @@ public static UShort leftShift(UShort x, long y) { * Overflow bits are masked to 32 bits. */ public static UInteger leftShift(UInteger x, long y) { - int shift = Math.floorMod(y, 32); + int shift = positiveModulo(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val << shift) & 0xFFFFFFFFL : (val >> shift) & 0xFFFFFFFFL; return UInteger.valueOf(val); @@ -3896,7 +3911,7 @@ public static UInteger leftShift(UInteger x, long y) { * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ public static ULong leftShift(ULong x, long y) { - int shift = Math.floorMod(y, 64); + int shift = positiveModulo(y, 64); long val = x.longValue(); val = (y >= 0) ? val << shift : val >> shift; return ULong.valueOf(val); @@ -3910,7 +3925,7 @@ public static ULong leftShift(ULong x, long y) { * @return the shifted integer */ public static int rightShift(int x, long y) { - int shift = Math.floorMod(y, 32); // normalize to 0~31 + int shift = positiveModulo(y, 32); // normalize to 0~31 return y >= 0 ? x >> shift : x << shift; // arithmetic right shift } @@ -3924,7 +3939,7 @@ public static int rightShift(int x, long y) { * @return the shifted long value */ public static long rightShift(long x, long y) { - int shift = Math.floorMod(y, 64); // normalize to 0~63 + int shift = positiveModulo(y, 64); // normalize to 0~63 return y >= 0 ? x >> shift : x << shift; } @@ -3946,7 +3961,7 @@ public static byte[] rightShift(byte[] bytes, long y) { // PostgreSQL behavior: always treat as right shift with modulo arithmetic // Negative y becomes equivalent positive shift - int shift = Math.floorMod(y, bitLen); + int shift = positiveModulo(y, bitLen); if (shift == 0) { return bytes.clone(); @@ -3993,7 +4008,7 @@ public static ByteString rightShift(ByteString bytes, long y) { * Overflow bits are masked to 8 bits. */ public static UByte rightShift(UByte x, long y) { - int shift = Math.floorMod(y, 8); + int shift = positiveModulo(y, 8); int val = x.byteValue() & 0xFF; val = (y >= 0) ? (val >> shift) & 0xFF : (val << shift) & 0xFF; return UByte.valueOf((byte) val); @@ -4004,7 +4019,7 @@ public static UByte rightShift(UByte x, long y) { * Overflow bits are masked to 16 bits. */ public static UShort rightShift(UShort x, long y) { - int shift = Math.floorMod(y, 16); + int shift = positiveModulo(y, 16); int val = x.shortValue() & 0xFFFF; val = (y >= 0) ? (val >> shift) & 0xFFFF : (val << shift) & 0xFFFF; return UShort.valueOf((short) val); @@ -4015,7 +4030,7 @@ public static UShort rightShift(UShort x, long y) { * Overflow bits are masked to 32 bits. */ public static UInteger rightShift(UInteger x, long y) { - int shift = Math.floorMod(y, 32); + int shift = positiveModulo(y, 32); long val = x.longValue() & 0xFFFFFFFFL; val = (y >= 0) ? (val >> shift) & 0xFFFFFFFFL : (val << shift) & 0xFFFFFFFFL; return UInteger.valueOf(val); @@ -4026,7 +4041,7 @@ public static UInteger rightShift(UInteger x, long y) { * Overflow bits are masked to 64 bits (long shifts naturally truncate). */ public static ULong rightShift(ULong x, long y) { - int shift = Math.floorMod(y, 64); + int shift = positiveModulo(y, 64); long val = x.longValue(); val = (y >= 0) ? val >> shift : val << shift; return ULong.valueOf(val); From 3191430d9a3118bcfc098f44f3fc0a911fde3b6a Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Thu, 9 Jul 2026 09:38:11 +0200 Subject: [PATCH 07/13] [CALCITE-7639] Right shift of BIGINT UNSIGNED sign-extends instead of shifting logically rightShift(ULong, long) shifted the raw signed long with an arithmetic '>>', so a BIGINT UNSIGNED value with the high bit set was sign-extended. For example, right-shifting 2^63 by 4 returned 17870283321406128128 instead of 576460752303423488. Unlike UByte/UShort/UInteger, which mask to a non-negative domain before shifting, ULong holds the full 64 bits, so the shift must be logical ('>>>'). leftShift(ULong, long) had the same defect in its negative-amount branch (which shifts right); both now use '>>>'. Also correct the reference documentation: a negative shift amount shifts in the opposite direction by the normalized magnitude, so RIGHTSHIFT(1, -2) is 1073741824 (a left shift by 30) and LEFTSHIFT(1, -2) is 0 (a right shift by 30). The previous wording, and the LEFTSHIFT(1, -2) example, were inaccurate. Add executing tests (run via CalciteSqlOperatorTest) for the high-bit BIGINT UNSIGNED shifts and for the negative-amount direction, which the existing tests did not distinguish. --- .../org/apache/calcite/runtime/SqlFunctions.java | 8 ++++++-- site/_docs/reference.md | 4 ++-- .../org/apache/calcite/test/SqlOperatorTest.java | 15 +++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index ec5802e1600d..6dcffd09b4e7 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3913,7 +3913,9 @@ public static UInteger leftShift(UInteger x, long y) { public static ULong leftShift(ULong x, long y) { int shift = positiveModulo(y, 64); long val = x.longValue(); - val = (y >= 0) ? val << shift : val >> shift; + // A negative shift amount shifts right; use a logical (unsigned) shift so + // the full-width ULong value is not sign-extended. + val = (y >= 0) ? val << shift : val >>> shift; return ULong.valueOf(val); } @@ -4043,7 +4045,9 @@ public static UInteger rightShift(UInteger x, long y) { public static ULong rightShift(ULong x, long y) { int shift = positiveModulo(y, 64); long val = x.longValue(); - val = (y >= 0) ? val >> shift : val << shift; + // Use a logical (unsigned) right shift: ULong holds the full 64 bits, so + // the raw long may be negative and an arithmetic '>>' would sign-extend. + val = (y >= 0) ? val >>> shift : val << shift; return ULong.valueOf(val); } diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 40c4de1192d5..38605d083dfb 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2973,8 +2973,8 @@ In the following: | * | BITAND(value1, value2) | Returns the bitwise AND of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITOR(value1, value2) | Returns the bitwise OR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITXOR(value1, value2) | Returns the bitwise XOR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. -| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). Negative shift amounts are converted to equivalent positive shifts through this modulo operation. For example, `LEFTSHIFT(1, -2)` returns `1073741824` (equivalent to `1 << 30`), and `LEFTSHIFT(8, -1)` returns `0` due to overflow. -| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). Negative shift amounts are converted to equivalent positive shifts through this modulo operation. For example, `RIGHTSHIFT(1024, 2)` returns `256` (equivalent to `1024 >> 2`), and `RIGHTSHIFT(-20, 2)` returns `-5`. +| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). The sign of *value2* selects the direction: a non-negative amount shifts left, and a negative amount shifts right by the normalized magnitude. For example, `LEFTSHIFT(1, -2)` returns `0` (a right shift by 30), and `LEFTSHIFT(8, -1)` returns `0`. +| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). The sign of *value2* selects the direction: a non-negative amount shifts right, and a negative amount shifts left by the normalized magnitude. For example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824` (a left shift by 30). | * | BITNOT(value) | Returns the bitwise NOT of *value*. *value* must be either an integer type or a binary value. | f | BITAND_AGG(value) | Equivalent to `BIT_AND(value)` | f | BITOR_AGG(value) | Equivalent to `BIT_OR(value)` diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 306957321b61..427f451c10f0 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -17006,8 +17006,16 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(1 AS INTEGER UNSIGNED) << 31", "2147483648", "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(1 AS INTEGER UNSIGNED) << -1", "0", "INTEGER UNSIGNED NOT NULL"); + // BIGINT UNSIGNED with the high bit set (2^63, built via 1 << 63), shifted + // left by a negative amount (i.e. right by 60): the implied right shift must + // be logical, not arithmetic (the raw long is negative). + f.checkScalar("CAST(1 AS BIGINT UNSIGNED) << 63 << -4", + "8", "BIGINT UNSIGNED NOT NULL"); // === Negative shift counts === + // A negative left shift shifts right by the normalized magnitude (here + // 32 - 2 = 30); it is not the same as a right shift by the given amount. + f.checkScalar("1 << -2", "0", "INTEGER NOT NULL"); // 1 >> 30 f.checkScalar("8 << -1", "0", "INTEGER NOT NULL"); f.checkScalar("16 << -2", "0", "INTEGER NOT NULL"); @@ -17208,6 +17216,10 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(2147483648 AS INTEGER UNSIGNED) >> 31", "1", "INTEGER UNSIGNED NOT NULL"); f.checkScalar("CAST(1 AS INTEGER UNSIGNED) >> -1", "2147483648", "INTEGER UNSIGNED NOT NULL"); + // BIGINT UNSIGNED with the high bit set (2^63, built via 1 << 63): the + // right shift must be logical, not arithmetic (the raw long is negative). + f.checkScalar("CAST(1 AS BIGINT UNSIGNED) << 63 >> 4", + "576460752303423488", "BIGINT UNSIGNED NOT NULL"); // A BIGINT shift amount is accepted (INTEGER family), but an unsigned shift // amount is not: the second operand must be a signed integer type. @@ -17219,6 +17231,9 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { false); // === Negative shift counts (normalized via modulo, then shifted the other way) === + // A negative right shift shifts left by the normalized magnitude (here + // 32 - 2 = 30); it is not the same as a left shift by the given amount. + f.checkScalar("1 >> -2", "1073741824", "INTEGER NOT NULL"); // 1 << 30 f.checkScalar("8 >> -1", "0", "INTEGER NOT NULL"); f.checkScalar("16 >> -2", "0", "INTEGER NOT NULL"); From 00954177734f4f67740a55bd77e5200e3eb067eb Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Thu, 9 Jul 2026 10:47:15 +0200 Subject: [PATCH 08/13] [CALCITE-7639] Re-trigger CI after a transient build-website failure From 7633feb5c3c8749a25e348c50c8d772389e0a88f Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Fri, 10 Jul 2026 08:07:38 +0200 Subject: [PATCH 09/13] [CALCITE-7639] Clarify binary shift semantics in the reference docs --- site/_docs/reference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 38605d083dfb..2d03277bb5eb 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2973,8 +2973,8 @@ In the following: | * | BITAND(value1, value2) | Returns the bitwise AND of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITOR(value1, value2) | Returns the bitwise OR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITXOR(value1, value2) | Returns the bitwise XOR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. -| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). The sign of *value2* selects the direction: a non-negative amount shifts left, and a negative amount shifts right by the normalized magnitude. For example, `LEFTSHIFT(1, -2)` returns `0` (a right shift by 30), and `LEFTSHIFT(8, -1)` returns `0`. -| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1*. For integers, this uses modulo 32; for binary types, it uses modulo (8 × byte_length). The sign of *value2* selects the direction: a non-negative amount shifts right, and a negative amount shifts left by the normalized magnitude. For example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824` (a left shift by 30). +| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts left and a negative amount shifts right by the normalized magnitude (for example, `LEFTSHIFT(1, -2)` returns `0`, a right shift by 30, and `LEFTSHIFT(8, -1)` returns `0`). For binary the shift is always to the left; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. +| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts right and a negative amount shifts left by the normalized magnitude (for example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824`, a left shift by 30). For binary the shift is always to the right; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. | * | BITNOT(value) | Returns the bitwise NOT of *value*. *value* must be either an integer type or a binary value. | f | BITAND_AGG(value) | Equivalent to `BIT_AND(value)` | f | BITOR_AGG(value) | Equivalent to `BIT_OR(value)` From 1c58b2e59e42ddc02742b6041c92f3e0c90d9e68 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Fri, 10 Jul 2026 08:07:38 +0200 Subject: [PATCH 10/13] [CALCITE-7639] Assert computed byte values in the shift unit tests --- .../apache/calcite/test/SqlFunctionsTest.java | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 3c8c5ff27dc6..8490be14099a 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -2045,19 +2045,30 @@ private long sqlTimestamp(String str) { return toLong(java.sql.Timestamp.valueOf(str)); } @Test void testLeftShift() { - // Test 1-byte array + // For every shift amount, cross-check the byte-array shift against the + // equivalent shift computed on the value's integer interpretation. The byte + // array is treated as a little-endian bit string of width 8 * length (index + // 0 is the least-significant byte), and leftShift always shifts left by the + // amount normalized modulo that width (so a negative amount wraps into range + // rather than reversing direction). byte[] data1 = {(byte) 0x0F}; // 00001111 for (int shift = -10; shift <= 10; shift++) { byte[] result = SqlFunctions.leftShift(data1.clone(), shift); - // Just verify it doesn't crash and returns correct length assertEquals(1, result.length); + int norm = Math.floorMod(shift, 8); + int expected = ((data1[0] & 0xFF) << norm) & 0xFF; + assertEquals((byte) expected, result[0]); } - // Test 2-byte array byte[] data2 = {(byte) 0x12, (byte) 0x34}; for (int shift = -18; shift <= 18; shift++) { byte[] result = SqlFunctions.leftShift(data2.clone(), shift); assertEquals(2, result.length); + int value = (data2[0] & 0xFF) | ((data2[1] & 0xFF) << 8); // little-endian + int norm = Math.floorMod(shift, 16); + int expected = (value << norm) & 0xFFFF; + assertEquals((byte) expected, result[0]); + assertEquals((byte) (expected >>> 8), result[1]); } // Verify specific known cases @@ -2069,19 +2080,30 @@ private long sqlTimestamp(String str) { } @Test void testRightShift() { - // Test 1-byte array + // For every shift amount, cross-check the byte-array shift against the + // equivalent shift computed on the value's integer interpretation. The byte + // array is treated as a little-endian bit string of width 8 * length (index + // 0 is the least-significant byte); rightShift always shifts right (zero + // fill) by the amount normalized modulo that width, so a negative amount + // wraps into range rather than reversing direction. byte[] data1 = {(byte) 0xF0}; // 11110000 for (int shift = -10; shift <= 10; shift++) { byte[] result = SqlFunctions.rightShift(data1.clone(), shift); - // Just verify it doesn't crash and returns correct length assertEquals(1, result.length); + int norm = Math.floorMod(shift, 8); + int expected = (data1[0] & 0xFF) >>> norm; + assertEquals((byte) expected, result[0]); } - // Test 2-byte array byte[] data2 = {(byte) 0x12, (byte) 0x34}; for (int shift = -18; shift <= 18; shift++) { byte[] result = SqlFunctions.rightShift(data2.clone(), shift); assertEquals(2, result.length); + int value = (data2[0] & 0xFF) | ((data2[1] & 0xFF) << 8); // little-endian + int norm = Math.floorMod(shift, 16); + int expected = value >>> norm; + assertEquals((byte) expected, result[0]); + assertEquals((byte) (expected >>> 8), result[1]); } // Verify specific known cases From d7fc15d9eea153cc1253cb86674c84b2bde4b97a Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Fri, 10 Jul 2026 08:07:38 +0200 Subject: [PATCH 11/13] [CALCITE-7639] Drop redundant BINARY casts and add VARBINARY shift tests --- .../apache/calcite/test/SqlOperatorTest.java | 105 ++++++++++++------ 1 file changed, 72 insertions(+), 33 deletions(-) diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 427f451c10f0..afaeaedf69c6 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -17242,23 +17242,40 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkScalar("0 >> 100", "0", "INTEGER NOT NULL"); // === Binary type tests === - f.checkScalar("CAST(X'FF' AS BINARY(1)) >> 1", "7f", "BINARY(1) NOT NULL"); - f.checkScalar("CAST(X'F0' AS BINARY(1)) >> 4", "0f", "BINARY(1) NOT NULL"); - f.checkScalar("CAST(X'08' AS BINARY(1)) >> 3", "01", "BINARY(1) NOT NULL"); - f.checkScalar("CAST(X'00' AS BINARY(1)) >> 5", "00", "BINARY(1) NOT NULL"); - - f.checkScalar("CAST(X'FFFF' AS BINARY(2)) >> 1", "ff7f", "BINARY(2) NOT NULL"); - f.checkScalar("CAST(X'1234' AS BINARY(2)) >> 4", "4103", "BINARY(2) NOT NULL"); - f.checkScalar("CAST(X'1234' AS BINARY(2)) >> 8", "3400", "BINARY(2) NOT NULL"); - - f.checkScalar("CAST(X'FF' AS BINARY(1)) >> 8", "ff", "BINARY(1) NOT NULL"); - f.checkScalar("CAST(X'FFFF' AS BINARY(2)) >> 16", "ffff", "BINARY(2) NOT NULL"); - - f.checkScalar("CAST(X'ABCD' AS BINARY(2)) >> 0", "abcd", "BINARY(2) NOT NULL"); - f.checkScalar("CAST(X'123456' AS BINARY(3)) >> 4", "416305", "BINARY(3) NOT NULL"); - f.checkScalar("CAST(X'0001' AS BINARY(2)) >> 1", "8000", "BINARY(2) NOT NULL"); - f.checkScalar("CAST(X'8000' AS BINARY(2)) >> 1", "4000", "BINARY(2) NOT NULL"); - f.checkScalar("CAST(X'F0' AS BINARY(1)) >> -4", "0f", "BINARY(1) NOT NULL"); + // A binary literal such as X'FF' already has type BINARY(1) (X'FFFF' is + // BINARY(2), and so on), so no cast is needed to exercise the BINARY path. + f.checkScalar("X'FF' >> 1", "7f", "BINARY(1) NOT NULL"); + f.checkScalar("X'F0' >> 4", "0f", "BINARY(1) NOT NULL"); + f.checkScalar("X'08' >> 3", "01", "BINARY(1) NOT NULL"); + f.checkScalar("X'00' >> 5", "00", "BINARY(1) NOT NULL"); + + f.checkScalar("X'FFFF' >> 1", "ff7f", "BINARY(2) NOT NULL"); + f.checkScalar("X'1234' >> 4", "4103", "BINARY(2) NOT NULL"); + f.checkScalar("X'1234' >> 8", "3400", "BINARY(2) NOT NULL"); + + f.checkScalar("X'FF' >> 8", "ff", "BINARY(1) NOT NULL"); + f.checkScalar("X'FFFF' >> 16", "ffff", "BINARY(2) NOT NULL"); + + f.checkScalar("X'ABCD' >> 0", "abcd", "BINARY(2) NOT NULL"); + f.checkScalar("X'123456' >> 4", "416305", "BINARY(3) NOT NULL"); + f.checkScalar("X'0001' >> 1", "8000", "BINARY(2) NOT NULL"); + f.checkScalar("X'8000' >> 1", "4000", "BINARY(2) NOT NULL"); + f.checkScalar("X'F0' >> -4", "0f", "BINARY(1) NOT NULL"); + + // === VARBINARY type tests === + // For variable-length binary the shift width is 8 * the actual value + // length, so the modulo used to normalize the amount tracks the value. + f.checkScalar("CAST(X'FF' AS VARBINARY) >> 1", "7f", "VARBINARY NOT NULL"); + f.checkScalar("CAST(X'F0' AS VARBINARY) >> 4", "0f", "VARBINARY NOT NULL"); + f.checkScalar("CAST(X'FFFF' AS VARBINARY) >> 1", "ff7f", "VARBINARY NOT NULL"); + f.checkScalar("CAST(X'123456' AS VARBINARY) >> 4", "416305", + "VARBINARY NOT NULL"); + // 8 % (8 * 1) = 0 leaves the 1-byte value unchanged, while 8 % (8 * 2) = 8 + // shifts the 2-byte value by a whole byte: the width follows the value. + f.checkScalar("CAST(X'FF' AS VARBINARY) >> 8", "ff", "VARBINARY NOT NULL"); + f.checkScalar("CAST(X'FFFF' AS VARBINARY) >> 8", "ff00", + "VARBINARY NOT NULL"); + f.checkScalar("CAST(X'F0' AS VARBINARY) >> -4", "0f", "VARBINARY NOT NULL"); // === Invalid argument types === f.checkFails("^1.2 >> 2^", @@ -17331,22 +17348,44 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkScalar("RIGHTSHIFT(0, 100)", "0", "INTEGER NOT NULL"); // === Binary types === - f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS BINARY(1)), 1)", "7f", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS BINARY(1)), 4)", "0f", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'08' AS BINARY(1)), 3)", "01", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'00' AS BINARY(1)), 5)", "00", "BINARY(1) NOT NULL"); - - f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS BINARY(2)), 1)", "ff7f", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'1234' AS BINARY(2)), 4)", "4103", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'1234' AS BINARY(2)), 8)", "3400", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS BINARY(1)), 8)", "ff", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS BINARY(2)), 16)", "ffff", "BINARY(2) NOT NULL"); - - f.checkScalar("RIGHTSHIFT(CAST(X'ABCD' AS BINARY(2)), 0)", "abcd", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'123456' AS BINARY(3)), 4)", "416305", "BINARY(3) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'0001' AS BINARY(2)), 1)", "8000", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'8000' AS BINARY(2)), 1)", "4000", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS BINARY(1)), -4)", "0f", "BINARY(1) NOT NULL"); + // A binary literal such as X'FF' already has type BINARY(1) (X'FFFF' is + // BINARY(2), and so on), so no cast is needed to exercise the BINARY path. + f.checkScalar("RIGHTSHIFT(X'FF', 1)", "7f", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'F0', 4)", "0f", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'08', 3)", "01", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'00', 5)", "00", "BINARY(1) NOT NULL"); + + f.checkScalar("RIGHTSHIFT(X'FFFF', 1)", "ff7f", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'1234', 4)", "4103", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'1234', 8)", "3400", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'FF', 8)", "ff", "BINARY(1) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'FFFF', 16)", "ffff", "BINARY(2) NOT NULL"); + + f.checkScalar("RIGHTSHIFT(X'ABCD', 0)", "abcd", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'123456', 4)", "416305", "BINARY(3) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'0001', 1)", "8000", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'8000', 1)", "4000", "BINARY(2) NOT NULL"); + f.checkScalar("RIGHTSHIFT(X'F0', -4)", "0f", "BINARY(1) NOT NULL"); + + // === VARBINARY types === + // For variable-length binary the shift width is 8 * the actual value + // length, so the modulo used to normalize the amount tracks the value. + f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS VARBINARY), 1)", "7f", + "VARBINARY NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS VARBINARY), 4)", "0f", + "VARBINARY NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS VARBINARY), 1)", "ff7f", + "VARBINARY NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'123456' AS VARBINARY), 4)", "416305", + "VARBINARY NOT NULL"); + // 8 % (8 * 1) = 0 leaves the 1-byte value unchanged, while 8 % (8 * 2) = 8 + // shifts the 2-byte value by a whole byte: the width follows the value. + f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS VARBINARY), 8)", "ff", + "VARBINARY NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS VARBINARY), 8)", "ff00", + "VARBINARY NOT NULL"); + f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS VARBINARY), -4)", "0f", + "VARBINARY NOT NULL"); // === Invalid types === f.checkFails("^RIGHTSHIFT(1.2, 2)^", "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", From 0095b5ac44c3a03daa776e06a22bb7140f0ccae8 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Mon, 13 Jul 2026 12:34:04 +0200 Subject: [PATCH 12/13] [CALCITE-7639] Descope binary right shift pending endianness design (CALCITE-7651) The byte[] shift added for LEFTSHIFT treats binary as little-endian and modulo-wraps the amount, which is inconsistent with the big-endian CAST(INT AS BINARY) from CALCITE-7368. Rather than cement that behaviour in a new operator, restrict >> and RIGHTSHIFT to integer/unsigned operands and reject BINARY/VARBINARY at validation. Binary shift semantics are tracked separately in CALCITE-7651; binary left shift is left as released. - Split the shared shift operand type checker: keep BINARY for <>/RIGHTSHIFT. - Remove the unused rightShift(byte[])/rightShift(ByteString) overloads. - Replace the binary/VARBINARY right-shift tests with checkFails that pin the rejection; keep the integer/unsigned coverage. - Update the RIGHTSHIFT reference docs accordingly. --- .../apache/calcite/runtime/SqlFunctions.java | 62 +--------- .../calcite/sql/fun/SqlStdOperatorTable.java | 24 +++- .../apache/calcite/test/SqlFunctionsTest.java | 37 +----- site/_docs/reference.md | 2 +- .../apache/calcite/test/SqlOperatorTest.java | 107 +++++------------- 5 files changed, 54 insertions(+), 178 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 6dcffd09b4e7..6b3948c0d5c4 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3945,65 +3945,9 @@ public static long rightShift(long x, long y) { return y >= 0 ? x >> shift : x << shift; } - /** - * Performs PostgresSQL-style bitwise shift on a byte array. - * Positive shift: right shift. - * Negative shift: treated as positive shift with modulo arithmetic. - * - * @param bytes the input byte array - * @param y the shift amount in bits - * @return the shifted byte array - */ - public static byte[] rightShift(byte[] bytes, long y) { - if (bytes.length == 0) { - return new byte[0]; - } - - int bitLen = bytes.length * 8; - - // PostgreSQL behavior: always treat as right shift with modulo arithmetic - // Negative y becomes equivalent positive shift - int shift = positiveModulo(y, bitLen); - - if (shift == 0) { - return bytes.clone(); - } - - byte[] result = new byte[bytes.length]; - - // Always perform right shift (even for originally negative y) - int byteShift = shift / 8; - int bitShift = shift % 8; - - for (int i = 0; i < bytes.length; i++) { - int srcIndex = i + byteShift; - int val = 0; - - // Get the main byte - if (srcIndex < bytes.length) { - val = (bytes[srcIndex] & 0xFF) >>> bitShift; - } - - // Get carry bits from next byte - if (srcIndex + 1 < bytes.length && bitShift != 0) { - val |= (bytes[srcIndex + 1] & 0xFF) << (8 - bitShift); - } - - result[i] = (byte) val; - } - return result; - } - - /** - * Performs PostgresSQL-style bitwise shift on ByteString. - * - * @param bytes the ByteString to shift - * @param y the shift amount in bits - * @return shifted ByteString - */ - public static ByteString rightShift(ByteString bytes, long y) { - return new ByteString(rightShift(bytes.getBytes(), y)); - } + // Right shift on binary (byte[]/ByteString) is intentionally not implemented: + // BINARY/VARBINARY operands are rejected for >> and RIGHTSHIFT until the + // endianness of bitwise shifts on binary is settled. See [CALCITE-7651]. /** * Performs PostgresSQL-style bitwise shift on UByte. diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 3e36b8c5d169..20a5e690ffe6 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -1369,16 +1369,28 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { new SqlBitOpAggFunction(SqlKind.BIT_XOR); /** - * Operand type checker shared by the shift operators ({@code <<}, {@code >>}) - * and their function forms ({@code LEFTSHIFT}, {@code RIGHTSHIFT}). The first - * operand is the value being shifted (integer, binary or unsigned numeric) and - * the second is the integer shift amount. + * Operand type checker shared by the left shift operator ({@code <<}) and its + * function form ({@code LEFTSHIFT}). The first operand is the value being + * shifted (integer, binary or unsigned numeric) and the second is the integer + * shift amount. */ private static final SqlOperandTypeChecker SHIFT_OPERAND_TYPE_CHECKER = OperandTypes.INTEGER_INTEGER .or(OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.INTEGER)) .or(OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)); + /** + * Operand type checker for the right shift operator ({@code >>}) and its + * function form ({@code RIGHTSHIFT}). Like {@link #SHIFT_OPERAND_TYPE_CHECKER} + * but the value being shifted may only be integer or unsigned numeric, not + * binary. Binary right shift is intentionally excluded until the endianness of + * bitwise shifts on {@code BINARY}/{@code VARBINARY} is settled; see + * [CALCITE-7651]. + */ + private static final SqlOperandTypeChecker NON_BINARY_SHIFT_OPERAND_TYPE_CHECKER = + OperandTypes.INTEGER_INTEGER + .or(OperandTypes.family(SqlTypeFamily.UNSIGNED_NUMERIC, SqlTypeFamily.INTEGER)); + /** * {@code <<} (left shift) operator. */ @@ -1413,7 +1425,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { true, ReturnTypes.ARG0_NULLABLE, InferTypes.FIRST_KNOWN, - SHIFT_OPERAND_TYPE_CHECKER); + NON_BINARY_SHIFT_OPERAND_TYPE_CHECKER); /** * right shift function. @@ -1423,7 +1435,7 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { "RIGHTSHIFT", SqlKind.OTHER_FUNCTION, ReturnTypes.ARG0_NULLABLE, - SHIFT_OPERAND_TYPE_CHECKER); + NON_BINARY_SHIFT_OPERAND_TYPE_CHECKER); //------------------------------------------------------------- // WINDOW Aggregate Functions diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index 8490be14099a..9231437cf8ea 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -2080,40 +2080,9 @@ private long sqlTimestamp(String str) { } @Test void testRightShift() { - // For every shift amount, cross-check the byte-array shift against the - // equivalent shift computed on the value's integer interpretation. The byte - // array is treated as a little-endian bit string of width 8 * length (index - // 0 is the least-significant byte); rightShift always shifts right (zero - // fill) by the amount normalized modulo that width, so a negative amount - // wraps into range rather than reversing direction. - byte[] data1 = {(byte) 0xF0}; // 11110000 - for (int shift = -10; shift <= 10; shift++) { - byte[] result = SqlFunctions.rightShift(data1.clone(), shift); - assertEquals(1, result.length); - int norm = Math.floorMod(shift, 8); - int expected = (data1[0] & 0xFF) >>> norm; - assertEquals((byte) expected, result[0]); - } - - byte[] data2 = {(byte) 0x12, (byte) 0x34}; - for (int shift = -18; shift <= 18; shift++) { - byte[] result = SqlFunctions.rightShift(data2.clone(), shift); - assertEquals(2, result.length); - int value = (data2[0] & 0xFF) | ((data2[1] & 0xFF) << 8); // little-endian - int norm = Math.floorMod(shift, 16); - int expected = value >>> norm; - assertEquals((byte) expected, result[0]); - assertEquals((byte) (expected >>> 8), result[1]); - } - - // Verify specific known cases - assertArrayEquals(new byte[]{(byte) 0xAB, (byte) 0xCD}, - SqlFunctions.rightShift(new byte[]{(byte) 0xAB, (byte) 0xCD}, 0)); - - assertArrayEquals(new byte[]{(byte) 0x40, (byte) 0x00}, - SqlFunctions.rightShift(new byte[]{(byte) 0x80, (byte) 0x00}, 1)); - - // Scalar arithmetic (sign-preserving) right shift + // Scalar arithmetic (sign-preserving) right shift. Binary right shift is + // intentionally not supported (see [CALCITE-7651]), so there is no + // byte-array overload to exercise here. assertEquals(2, SqlFunctions.rightShift(8, 2)); assertEquals(-5, SqlFunctions.rightShift(-20, 2)); } diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 2d03277bb5eb..0569b1a54ee1 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2974,7 +2974,7 @@ In the following: | * | BITOR(value1, value2) | Returns the bitwise OR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITXOR(value1, value2) | Returns the bitwise XOR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts left and a negative amount shifts right by the normalized magnitude (for example, `LEFTSHIFT(1, -2)` returns `0`, a right shift by 30, and `LEFTSHIFT(8, -1)` returns `0`). For binary the shift is always to the left; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. -| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts right and a negative amount shifts left by the normalized magnitude (for example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824`, a left shift by 30). For binary the shift is always to the right; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. +| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer or unsigned integer (binary right shift is not yet supported). The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`). The sign of *value2* selects the direction: a non-negative amount shifts right and a negative amount shifts left by the normalized magnitude (for example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824`, a left shift by 30). | * | BITNOT(value) | Returns the bitwise NOT of *value*. *value* must be either an integer type or a binary value. | f | BITAND_AGG(value) | Equivalent to `BIT_AND(value)` | f | BITOR_AGG(value) | Equivalent to `BIT_OR(value)` diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index afaeaedf69c6..edd5636fb428 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -17226,7 +17226,7 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkScalar("CAST(8 AS INTEGER) >> CAST(2 AS BIGINT)", "2", "INTEGER NOT NULL"); f.checkFails("^8 >> CAST(2 AS INTEGER UNSIGNED)^", "Cannot apply '>>' to arguments of type ' >> '\\. " - + "Supported form\\(s\\): ' >> '\\n' >> '\\n" + + "Supported form\\(s\\): ' >> '\\n" + "' >> '", false); @@ -17241,46 +17241,25 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkScalar("0 >> 32", "0", "INTEGER NOT NULL"); f.checkScalar("0 >> 100", "0", "INTEGER NOT NULL"); - // === Binary type tests === - // A binary literal such as X'FF' already has type BINARY(1) (X'FFFF' is - // BINARY(2), and so on), so no cast is needed to exercise the BINARY path. - f.checkScalar("X'FF' >> 1", "7f", "BINARY(1) NOT NULL"); - f.checkScalar("X'F0' >> 4", "0f", "BINARY(1) NOT NULL"); - f.checkScalar("X'08' >> 3", "01", "BINARY(1) NOT NULL"); - f.checkScalar("X'00' >> 5", "00", "BINARY(1) NOT NULL"); - - f.checkScalar("X'FFFF' >> 1", "ff7f", "BINARY(2) NOT NULL"); - f.checkScalar("X'1234' >> 4", "4103", "BINARY(2) NOT NULL"); - f.checkScalar("X'1234' >> 8", "3400", "BINARY(2) NOT NULL"); - - f.checkScalar("X'FF' >> 8", "ff", "BINARY(1) NOT NULL"); - f.checkScalar("X'FFFF' >> 16", "ffff", "BINARY(2) NOT NULL"); - - f.checkScalar("X'ABCD' >> 0", "abcd", "BINARY(2) NOT NULL"); - f.checkScalar("X'123456' >> 4", "416305", "BINARY(3) NOT NULL"); - f.checkScalar("X'0001' >> 1", "8000", "BINARY(2) NOT NULL"); - f.checkScalar("X'8000' >> 1", "4000", "BINARY(2) NOT NULL"); - f.checkScalar("X'F0' >> -4", "0f", "BINARY(1) NOT NULL"); - - // === VARBINARY type tests === - // For variable-length binary the shift width is 8 * the actual value - // length, so the modulo used to normalize the amount tracks the value. - f.checkScalar("CAST(X'FF' AS VARBINARY) >> 1", "7f", "VARBINARY NOT NULL"); - f.checkScalar("CAST(X'F0' AS VARBINARY) >> 4", "0f", "VARBINARY NOT NULL"); - f.checkScalar("CAST(X'FFFF' AS VARBINARY) >> 1", "ff7f", "VARBINARY NOT NULL"); - f.checkScalar("CAST(X'123456' AS VARBINARY) >> 4", "416305", - "VARBINARY NOT NULL"); - // 8 % (8 * 1) = 0 leaves the 1-byte value unchanged, while 8 % (8 * 2) = 8 - // shifts the 2-byte value by a whole byte: the width follows the value. - f.checkScalar("CAST(X'FF' AS VARBINARY) >> 8", "ff", "VARBINARY NOT NULL"); - f.checkScalar("CAST(X'FFFF' AS VARBINARY) >> 8", "ff00", - "VARBINARY NOT NULL"); - f.checkScalar("CAST(X'F0' AS VARBINARY) >> -4", "0f", "VARBINARY NOT NULL"); + // === Binary operands are not supported === + // Unlike '<<', binary right shift is intentionally rejected until the + // endianness of bitwise shifts on binary is settled (see [CALCITE-7651]). + // A binary literal such as X'FF' already has type BINARY(1). + f.checkFails("^X'FF' >> 1^", + "Cannot apply '>>' to arguments of type ' >> '\\. " + + "Supported form\\(s\\): ' >> '\\n" + + "' >> '", + false); + f.checkFails("^CAST(X'FF' AS VARBINARY) >> 1^", + "Cannot apply '>>' to arguments of type ' >> '\\. " + + "Supported form\\(s\\): ' >> '\\n" + + "' >> '", + false); // === Invalid argument types === f.checkFails("^1.2 >> 2^", "Cannot apply '>>' to arguments of type ' >> '\\. Supported " - + "form\\(s\\): ' >> '\\n' >> '\\n' " + + "form\\(s\\): ' >> '\\n' " + ">> '", false); @@ -17347,54 +17326,26 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { f.checkScalar("RIGHTSHIFT(0, 32)", "0", "INTEGER NOT NULL"); f.checkScalar("RIGHTSHIFT(0, 100)", "0", "INTEGER NOT NULL"); - // === Binary types === - // A binary literal such as X'FF' already has type BINARY(1) (X'FFFF' is - // BINARY(2), and so on), so no cast is needed to exercise the BINARY path. - f.checkScalar("RIGHTSHIFT(X'FF', 1)", "7f", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'F0', 4)", "0f", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'08', 3)", "01", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'00', 5)", "00", "BINARY(1) NOT NULL"); - - f.checkScalar("RIGHTSHIFT(X'FFFF', 1)", "ff7f", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'1234', 4)", "4103", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'1234', 8)", "3400", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'FF', 8)", "ff", "BINARY(1) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'FFFF', 16)", "ffff", "BINARY(2) NOT NULL"); - - f.checkScalar("RIGHTSHIFT(X'ABCD', 0)", "abcd", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'123456', 4)", "416305", "BINARY(3) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'0001', 1)", "8000", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'8000', 1)", "4000", "BINARY(2) NOT NULL"); - f.checkScalar("RIGHTSHIFT(X'F0', -4)", "0f", "BINARY(1) NOT NULL"); - - // === VARBINARY types === - // For variable-length binary the shift width is 8 * the actual value - // length, so the modulo used to normalize the amount tracks the value. - f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS VARBINARY), 1)", "7f", - "VARBINARY NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS VARBINARY), 4)", "0f", - "VARBINARY NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS VARBINARY), 1)", "ff7f", - "VARBINARY NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'123456' AS VARBINARY), 4)", "416305", - "VARBINARY NOT NULL"); - // 8 % (8 * 1) = 0 leaves the 1-byte value unchanged, while 8 % (8 * 2) = 8 - // shifts the 2-byte value by a whole byte: the width follows the value. - f.checkScalar("RIGHTSHIFT(CAST(X'FF' AS VARBINARY), 8)", "ff", - "VARBINARY NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'FFFF' AS VARBINARY), 8)", "ff00", - "VARBINARY NOT NULL"); - f.checkScalar("RIGHTSHIFT(CAST(X'F0' AS VARBINARY), -4)", "0f", - "VARBINARY NOT NULL"); + // === Binary operands are not supported === + // Unlike LEFTSHIFT, binary right shift is intentionally rejected until the + // endianness of bitwise shifts on binary is settled (see [CALCITE-7651]). + // A binary literal such as X'FF' already has type BINARY(1). + f.checkFails("^RIGHTSHIFT(X'FF', 1)^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + f.checkFails("^RIGHTSHIFT(CAST(X'FF' AS VARBINARY), 1)^", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + false); + // === Invalid types === f.checkFails("^RIGHTSHIFT(1.2, 2)^", - "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", false); // A BIGINT shift amount is accepted, but an unsigned shift amount is not: // the second argument must be a signed integer type. f.checkScalar("RIGHTSHIFT(8, CAST(2 AS BIGINT))", "2", "INTEGER NOT NULL"); f.checkFails("^RIGHTSHIFT(8, CAST(2 AS INTEGER UNSIGNED))^", - "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", + "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", false); From dc399d61907d51424cbd7cb2c9000c1dd23c8d89 Mon Sep 17 00:00:00 2001 From: Niels Pardon Date: Mon, 13 Jul 2026 14:08:25 +0200 Subject: [PATCH 13/13] [CALCITE-7639] Right-shift comment and shift-modulo docs are inaccurate --- .../org/apache/calcite/adapter/enumerable/RexImpTable.java | 3 ++- .../main/java/org/apache/calcite/runtime/SqlFunctions.java | 2 -- site/_docs/reference.md | 4 ++-- .../main/java/org/apache/calcite/test/SqlOperatorTest.java | 1 - 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index b7d0456fac64..57edf126a5c4 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -923,7 +923,8 @@ void populate1() { defineMethod(BIT_LEFT_SHIFT, BuiltInMethod.LEFT_SHIFT.method, NullPolicy.STRICT); // Right shift operations: shift bits to the right by specified amount. - // Supports integer, unsigned integer, and binary data types. + // Supports integer and unsigned integer data types. Binary right shift is + // intentionally not supported; see [CALCITE-7651]. // Shift amount is normalized using modulo arithmetic based on data type bit width. // RIGHTSHIFT: Function call syntax for bitwise right shift operation (e.g., RIGHTSHIFT(x, y)) diff --git a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java index 6b3948c0d5c4..41fc4484b85d 100644 --- a/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/SqlFunctions.java @@ -3931,8 +3931,6 @@ public static int rightShift(int x, long y) { return y >= 0 ? x >> shift : x << shift; // arithmetic right shift } - - // ----------------- long ----------------- /** * Performs PostgresSQL-style bitwise shift on a 64-bit long value. * diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 0569b1a54ee1..e6d58eb84f62 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2973,8 +2973,8 @@ In the following: | * | BITAND(value1, value2) | Returns the bitwise AND of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITOR(value1, value2) | Returns the bitwise OR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. | * | BITXOR(value1, value2) | Returns the bitwise XOR of *value1* and *value2*. *value1* and *value2* must both be integer or binary values. Binary values must be of the same length. -| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts left and a negative amount shifts right by the normalized magnitude (for example, `LEFTSHIFT(1, -2)` returns `0`, a right shift by 30, and `LEFTSHIFT(8, -1)` returns `0`). For binary the shift is always to the left; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. -| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer or unsigned integer (binary right shift is not yet supported). The shift amount *value2* is normalized using modulo arithmetic based on the bit width of *value1* (for example, modulo 32 for `INTEGER` and modulo 64 for `BIGINT`). The sign of *value2* selects the direction: a non-negative amount shifts right and a negative amount shifts left by the normalized magnitude (for example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824`, a left shift by 30). +| * | LEFTSHIFT(value1, value2) | Returns the result of left-shifting *value1* by *value2* bits. *value1* can be integer, unsigned integer, or binary. For binary, the result has the same length as *value1*. The shift amount *value2* is normalized using modulo arithmetic: for signed integer types the modulus is 32 for `TINYINT`, `SMALLINT` and `INTEGER` (all backed by a 32-bit representation) and 64 for `BIGINT`; for unsigned integer types it matches the type's bit width (modulo 8, 16, 32 or 64); for binary types it is modulo (8 × N), where N is the actual length in bytes of the *value1* value — for a variable-length `VARBINARY` value this is the length of the value itself, not its declared maximum. For integer and unsigned types the sign of *value2* selects the direction: a non-negative amount shifts left and a negative amount shifts right by the normalized magnitude (for example, `LEFTSHIFT(1, -2)` returns `0`, a right shift by 30, and `LEFTSHIFT(8, -1)` returns `0`). For binary the shift is always to the left; a negative *value2* is simply folded into the range [0, 8 × N) by the same modulo. +| * | RIGHTSHIFT(value1, value2) | Returns the result of right-shifting *value1* by *value2* bits. For signed integers the shift is arithmetic (the sign bit is preserved). *value1* can be integer or unsigned integer (binary right shift is not yet supported). The shift amount *value2* is normalized using modulo arithmetic: for signed integer types the modulus is 32 for `TINYINT`, `SMALLINT` and `INTEGER` (all backed by a 32-bit representation) and 64 for `BIGINT`; for unsigned integer types it matches the type's bit width (modulo 8, 16, 32 or 64). The sign of *value2* selects the direction: a non-negative amount shifts right and a negative amount shifts left by the normalized magnitude (for example, `RIGHTSHIFT(1024, 2)` returns `256`, `RIGHTSHIFT(-20, 2)` returns `-5`, and `RIGHTSHIFT(1, -2)` returns `1073741824`, a left shift by 30). | * | BITNOT(value) | Returns the bitwise NOT of *value*. *value* must be either an integer type or a binary value. | f | BITAND_AGG(value) | Equivalent to `BIT_AND(value)` | f | BITOR_AGG(value) | Equivalent to `BIT_OR(value)` diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index edd5636fb428..243bfca489e5 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -17348,7 +17348,6 @@ private static void checkLogicalOrFunc(SqlOperatorFixture f) { "Cannot apply 'RIGHTSHIFT' to arguments of type 'RIGHTSHIFT\\(, \\)'\\. Supported form\\(s\\): 'RIGHTSHIFT\\(, \\)'\\n'RIGHTSHIFT\\(, \\)'", false); - // === Nulls === f.checkNull("RIGHTSHIFT(CAST(NULL AS INTEGER), 5)"); f.checkNull("RIGHTSHIFT(10, CAST(NULL AS INTEGER))");