Table: Ignore null attribute values in insert (#17790)

Author: Caideyipi

integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java

@@ -1018,6 +1018,41 @@ public void testInsertSingleColumn() throws SQLException, InterruptedException {
     }
   }
 
+  @Test
+  public void testInsertNullAttributeDoesNotOverwriteDeviceAttribute() throws SQLException {
+    try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+        Statement statement = connection.createStatement()) {
+      statement.execute("use \"test\"");
+      statement.execute(
+          "create table attr_null_insert(tag1 string tag, attr1 string attribute, attr2 string attribute, s1 int32 field)");
+
+      statement.execute(
+          "insert into attr_null_insert(time, tag1, attr1, attr2, s1) values(1, 'd1', 'old1', 'old2', 1)");
+      statement.execute(
+          "insert into attr_null_insert(time, tag1, attr1, attr2, s1) values(2, 'd1', null, null, 2)");
+      assertDeviceAttributes(statement, "old1", "old2");
+
+      statement.execute(
+          "insert into attr_null_insert(time, tag1, attr1, attr2, s1) values(3, 'd1', null, 'new2', 3)");
+      assertDeviceAttributes(statement, "old1", "new2");
+
+      statement.execute("update attr_null_insert set attr1 = null where tag1 = 'd1'");
+      assertDeviceAttributes(statement, null, "new2");
+    }
+  }
+
+  private void assertDeviceAttributes(
+      final Statement statement, final String expectedAttr1, final String expectedAttr2)
+      throws SQLException {
+    try (final ResultSet resultSet = statement.executeQuery("show devices from attr_null_insert")) {
+      assertTrue(resultSet.next());
+      assertEquals("d1", resultSet.getString("tag1"));
+      assertEquals(expectedAttr1, resultSet.getString("attr1"));
+      assertEquals(expectedAttr2, resultSet.getString("attr2"));
+      assertFalse(resultSet.next());
+    }
+  }
+
   @Test
   public void testInsertWithTTL() {
     try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java

@@ -36,6 +36,7 @@
 
 import org.apache.tsfile.file.metadata.IDeviceID;
 import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Constants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -74,7 +75,9 @@ public void validateDeviceSchema(
     // High-cost operations, shall only be called once
     final List<Object[]> deviceIdList = schemaValidation.getDeviceIdList();
     final List<String> attributeKeyList = schemaValidation.getAttributeColumnNameList();
-    final List<Object[]> attributeValueList = schemaValidation.getAttributeValueList();
+    // In normal inserts, null attributes mean no-op. Raw null is reserved for explicit clears.
+    final List<Object[]> attributeValueList =
+        normalizeNullAttributeValuesForInsert(schemaValidation.getAttributeValueList());
 
     if (LOGGER.isDebugEnabled()) {
       LOGGER.debug(
@@ -194,13 +197,52 @@ static boolean isAttributeUpdateRequired(
     for (int j = 0, size = attributeKeyList.size(); j < size; j++) {
       final Object inputValue =
           deviceAttributeValueList.length > j ? deviceAttributeValueList[j] : null;
+      if (inputValue == null || inputValue == Constants.NONE) {
+        continue;
+      }
       if (!Objects.equals(attributeMap.get(attributeKeyList.get(j)), inputValue)) {
         return true;
       }
     }
     return false;
   }
 
+  private List<Object[]> normalizeNullAttributeValuesForInsert(
+      final List<Object[]> attributeValueList) {
+    List<Object[]> result = null;
+    for (int i = 0; i < attributeValueList.size(); i++) {
+      final Object[] deviceAttributeValueList = attributeValueList.get(i);
+      final Object[] normalizedAttributeValueList =
+          normalizeNullAttributeValuesForInsert(deviceAttributeValueList);
+      if (result != null) {
+        result.add(normalizedAttributeValueList);
+      } else if (normalizedAttributeValueList != deviceAttributeValueList) {
+        result = new ArrayList<>(attributeValueList.size());
+        for (int j = 0; j < i; j++) {
+          result.add(attributeValueList.get(j));
+        }
+        result.add(normalizedAttributeValueList);
+      }
+    }
+    return result == null ? attributeValueList : result;
+  }
+
+  private Object[] normalizeNullAttributeValuesForInsert(final Object[] deviceAttributeValueList) {
+    for (int i = 0; i < deviceAttributeValueList.length; i++) {
+      if (deviceAttributeValueList[i] == null) {
+        final Object[] result =
+            Arrays.copyOf(deviceAttributeValueList, deviceAttributeValueList.length);
+        for (int j = i; j < result.length; j++) {
+          if (result[j] == null) {
+            result[j] = Constants.NONE;
+          }
+        }
+        return result;
+      }
+    }
+    return deviceAttributeValueList;
+  }
+
   private void autoCreateOrUpdateDeviceSchema(
       final ITableDeviceSchemaValidation schemaValidation,
       final ValidateResult previousValidateResult,

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java

@@ -90,20 +90,27 @@ public List<String> getAttributeColumnNameList() {
   public List<Object[]> getAttributeValueList() {
     prepareDeviceID2LastIdxMap();
     final InsertTabletStatement insertTabletStatement = getInnerTreeStatement();
-    List<Object[]> result = new ArrayList<>(insertTabletStatement.getRowCount());
     final List<Integer> attrColumnIndices = insertTabletStatement.getAttrColumnIndices();
-    for (Integer rowIndex : deviceID2LastIdxMap.values()) {
-      Object[] attrValues = new Object[attrColumnIndices.size()];
+
+    final Map<IDeviceID, Object[]> deviceID2AttributeValues =
+        new LinkedHashMap<>(deviceID2LastIdxMap.size());
+    for (final IDeviceID deviceID : deviceID2LastIdxMap.keySet()) {
+      deviceID2AttributeValues.put(deviceID, new Object[attrColumnIndices.size()]);
+    }
+
+    for (int rowIndex = 0; rowIndex < insertTabletStatement.getRowCount(); rowIndex++) {
+      final Object[] attrValues =
+          deviceID2AttributeValues.get(insertTabletStatement.getTableDeviceID(rowIndex));
       for (int attrColNum = 0; attrColNum < attrColumnIndices.size(); attrColNum++) {
         final int columnIndex = attrColumnIndices.get(attrColNum);
         if (!insertTabletStatement.isNull(rowIndex, columnIndex)) {
           attrValues[attrColNum] =
               ((Object[]) insertTabletStatement.getColumns()[columnIndex])[rowIndex];
         }
       }
-      result.add(attrValues);
     }
-    return result;
+
+    return new ArrayList<>(deviceID2AttributeValues.values());
   }
 
   // The map cannot be maintained during construction because the IDeviceID may be reset later.

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidatorTest.java

@@ -20,6 +20,7 @@
 package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher;
 
 import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Constants;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -32,17 +33,39 @@
 public class TableDeviceSchemaValidatorTest {
 
   @Test
-  public void testNullAttributeNeedsUpdate() {
+  public void testNullAttributeSkipsUpdate() {
     final Map<String, Binary> attributeMap = new HashMap<>();
     attributeMap.put("attr", new Binary("x", StandardCharsets.UTF_8));
 
-    Assert.assertTrue(
+    Assert.assertFalse(
         TableDeviceSchemaValidator.isAttributeUpdateRequired(
             Collections.singletonList("attr"), new Object[] {null}, attributeMap));
   }
 
   @Test
-  public void testMissingTailAttributeValueEqualsNull() {
+  public void testNoneAttributeSkipsUpdate() {
+    final Map<String, Binary> attributeMap = new HashMap<>();
+    attributeMap.put("attr", new Binary("x", StandardCharsets.UTF_8));
+
+    Assert.assertFalse(
+        TableDeviceSchemaValidator.isAttributeUpdateRequired(
+            Collections.singletonList("attr"), new Object[] {Constants.NONE}, attributeMap));
+  }
+
+  @Test
+  public void testNonNullAttributeNeedsUpdate() {
+    final Map<String, Binary> attributeMap = new HashMap<>();
+    attributeMap.put("attr", new Binary("x", StandardCharsets.UTF_8));
+
+    Assert.assertTrue(
+        TableDeviceSchemaValidator.isAttributeUpdateRequired(
+            Collections.singletonList("attr"),
+            new Object[] {new Binary("y", StandardCharsets.UTF_8)},
+            attributeMap));
+  }
+
+  @Test
+  public void testMissingTailAttributeValueSkipsUpdate() {
     final Map<String, Binary> attributeMap = new HashMap<>();
     attributeMap.put("attr1", new Binary("x", StandardCharsets.UTF_8));
 

iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTabletTest.java

@@ -127,4 +127,73 @@ public void testWithNull() {
     assertArrayEquals(new Object[] {"id3_1"}, deviceIdList.get(2));
     assertArrayEquals(new Object[] {}, deviceIdList.get(3));
   }
+
+  @Test
+  public void testDuplicateDeviceUsesLastNonNullAttributeValue() {
+    InsertTabletStatement innerStmt = new InsertTabletStatement();
+    innerStmt.setDevicePath(new PartialPath("table1", false));
+    innerStmt.setTimes(new long[] {1, 2, 3, 4});
+    innerStmt.setRowCount(4);
+    innerStmt.setMeasurements(new String[] {"deviceId", "attr1", "attr2", "measurement"});
+    innerStmt.setColumnCategories(
+        new TsTableColumnCategory[] {
+          TsTableColumnCategory.TAG,
+          TsTableColumnCategory.ATTRIBUTE,
+          TsTableColumnCategory.ATTRIBUTE,
+          TsTableColumnCategory.FIELD
+        });
+    innerStmt.setDataTypes(
+        new TSDataType[] {
+          TSDataType.STRING, TSDataType.STRING, TSDataType.STRING, TSDataType.STRING
+        });
+    innerStmt.setColumns(
+        new Object[] {
+          new Binary[] {
+            new Binary("d1", StandardCharsets.UTF_8),
+            new Binary("d1", StandardCharsets.UTF_8),
+            new Binary("d1", StandardCharsets.UTF_8),
+            new Binary("d1", StandardCharsets.UTF_8)
+          },
+          new Binary[] {
+            new Binary("attr1_1", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+            new Binary("attr1_3", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE
+          },
+          new Binary[] {
+            new Binary("attr2_1", StandardCharsets.UTF_8),
+            new Binary("attr2_2", StandardCharsets.UTF_8),
+            Binary.EMPTY_VALUE,
+            Binary.EMPTY_VALUE
+          },
+          new Binary[] {
+            new Binary("m1", StandardCharsets.UTF_8),
+            new Binary("m2", StandardCharsets.UTF_8),
+            new Binary("m3", StandardCharsets.UTF_8),
+            new Binary("m4", StandardCharsets.UTF_8)
+          },
+        });
+    innerStmt.setBitMaps(
+        new BitMap[] {
+          new BitMap(4, new byte[] {0x00}),
+          new BitMap(4, new byte[] {1 << 1 | 1 << 3}),
+          new BitMap(4, new byte[] {1 << 2 | 1 << 3}),
+          new BitMap(4, new byte[] {0x00}),
+        });
+
+    InsertTablet insertTablet = new InsertTablet(innerStmt, null);
+    assertEquals(Arrays.asList("attr1", "attr2"), insertTablet.getAttributeColumnNameList());
+    List<Object[]> attributeValueList = insertTablet.getAttributeValueList();
+    assertEquals(1, attributeValueList.size());
+    assertArrayEquals(
+        new Object[] {
+          new Binary("attr1_3", StandardCharsets.UTF_8),
+          new Binary("attr2_2", StandardCharsets.UTF_8)
+        },
+        attributeValueList.get(0));
+
+    List<Object[]> deviceIdList = insertTablet.getDeviceIdList();
+    assertEquals(1, deviceIdList.size());
+    assertArrayEquals(new Object[] {"d1"}, deviceIdList.get(0));
+  }
 }