encrypt key generate and destroy (#16176)

* add save encrypt parameter to common config

* add user_encrypt_token_hint

* delete the save_encrypt_key parameter

* update tsfile version

* set key to zero when exiting datanode

* remove some noted code

Author: zhujt20

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java

@@ -103,12 +103,6 @@ public CommonConfig setEncryptType(String encryptType) {
     return this;
   }
 
-  @Override
-  public CommonConfig setEncryptKeyPath(String encryptKeyPath) {
-    setProperty("encrypt_key_path", encryptKeyPath);
-    return this;
-  }
-
   @Override
   public CommonConfig setEnableGrantOption(boolean enableGrantOption) {
     setProperty("enable_grant_option", String.valueOf(enableGrantOption));

integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java

@@ -82,13 +82,6 @@ public CommonConfig setEncryptType(String encryptType) {
     return this;
   }
 
-  @Override
-  public CommonConfig setEncryptKeyPath(String encryptKeyPath) {
-    cnConfig.setProperty("encrypt_key_path", encryptKeyPath);
-    dnConfig.setProperty("encrypt_key_path", encryptKeyPath);
-    return this;
-  }
-
   @Override
   public CommonConfig setEnableGrantOption(boolean enableGrantOption) {
     cnConfig.setEnableGrantOption(enableGrantOption);

integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java

@@ -59,11 +59,6 @@ public CommonConfig setEncryptType(String encryptType) {
     return this;
   }
 
-  @Override
-  public CommonConfig setEncryptKeyPath(String encryptKeyPath) {
-    return this;
-  }
-
   @Override
   public CommonConfig setEnableGrantOption(boolean enableGrantOption) {
     return this;

integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java

@@ -38,8 +38,6 @@ public interface CommonConfig {
 
   CommonConfig setEncryptType(String encryptType);
 
-  CommonConfig setEncryptKeyPath(String encryptKeyPath);
-
   CommonConfig setEnableGrantOption(boolean enableGrantOption);
 
   CommonConfig setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(int maxMs);

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java

@@ -30,6 +30,7 @@
 import org.apache.iotdb.db.storageengine.rescon.disk.DirectoryChecker;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.tsfile.common.conf.TSFileConfig;
 import org.apache.tsfile.common.conf.TSFileDescriptor;
 import org.apache.tsfile.encrypt.EncryptUtils;
 import org.apache.tsfile.exception.encrypt.EncryptException;
@@ -78,7 +79,8 @@ public class IoTDBStartCheck {
   private static final String SCHEMA_REGION_CONSENSUS_PORT = "dn_schema_region_consensus_port";
   private static final String DATA_REGION_CONSENSUS_PORT = "dn_data_region_consensus_port";
   private static final String ENCRYPT_MAGIC_STRING = "encrypt_magic_string";
-
+  private static final String ENCRYPT_SALT = "encrypt_salt";
+  private static final String ENCRYPT_TOKEN_HINT = "encrypt_token_hint";
   private static final String magicString = "thisisusedfortsfileencrypt";
 
   // Mutable system parameters
@@ -316,11 +318,32 @@ public void serializeEncryptMagicString() throws IOException {
         throw new EncryptException(
             "encryptType is not UNENCRYPTED, but user_encrypt_token is not set. Please set it in the environment variable.");
       }
+      String tokenHint = System.getenv("user_encrypt_token_hint");
+      if (tokenHint != null && !tokenHint.trim().isEmpty()) {
+        // If user_encrypt_token_hint is set, it should follow some rules.
+        // For example, it could not include user_encrypt_token.
+        if (tokenHint.toLowerCase().contains(token.toLowerCase())) {
+          throw new EncryptException(
+              "user_encrypt_token_hint should not include user_encrypt_token, please check it in your environment variable.");
+        }
+        if (tokenHint
+            .toLowerCase()
+            .contains(new StringBuilder(token.toLowerCase()).reverse().toString())) {
+          throw new EncryptException(
+              "user_encrypt_token_hint should not include the reverse of user_encrypt_token, please check it in your environment variable.");
+        }
+      }
     }
     String encryptMagicString =
         EncryptUtils.byteArrayToHexString(
-            TSFileDescriptor.getInstance().getConfig().getEncryptKey());
+            EncryptUtils.getEncrypt().getEncryptor().encrypt(magicString.getBytes()));
     systemProperties.put(ENCRYPT_MAGIC_STRING, () -> encryptMagicString);
+    String encryptSalt =
+        EncryptUtils.byteArrayToHexString(
+            TSFileDescriptor.getInstance().getConfig().getEncryptSalt());
+    systemProperties.put(ENCRYPT_SALT, () -> encryptSalt);
+    String encryptTokenHint = CommonDescriptor.getInstance().getConfig().getUserEncryptTokenHint();
+    systemProperties.put(ENCRYPT_TOKEN_HINT, () -> encryptTokenHint);
     generateOrOverwriteSystemPropertiesFile();
   }
 
@@ -360,10 +383,36 @@ public void generateOrOverwriteSystemPropertiesFile() throws IOException {
 
   public void checkEncryptMagicString() throws IOException, ConfigurationException {
     properties = systemPropertiesHandler.read();
-    String encryptMagicString = properties.getProperty("encrypt_magic_string");
-    if (encryptMagicString != null) {
-      byte[] magicBytes = EncryptUtils.hexStringToByteArray(encryptMagicString);
-      TSFileDescriptor.getInstance().getConfig().setEncryptKey(magicBytes);
+    CommonDescriptor.getInstance()
+        .getConfig()
+        .setUserEncryptTokenHint(properties.getProperty(ENCRYPT_TOKEN_HINT));
+    String encryptSalt = properties.getProperty(ENCRYPT_SALT);
+    byte[] saltBytes = EncryptUtils.hexStringToByteArray(encryptSalt);
+    TSFileDescriptor.getInstance().getConfig().setEncryptSalt(saltBytes);
+
+    if (!Objects.equals(TSFileDescriptor.getInstance().getConfig().getEncryptType(), "UNENCRYPTED")
+        && !Objects.equals(
+            TSFileDescriptor.getInstance().getConfig().getEncryptType(),
+            "org.apache.tsfile.encrypt.UNENCRYPTED")) {
+      String token = System.getenv("user_encrypt_token");
+      if (token == null || token.trim().isEmpty()) {
+        throw new EncryptException(
+            "restart system after not storing key, but user_encrypt_token is not set. Please set it in the environment variable before restart. Here is your token hint info: "
+                + CommonDescriptor.getInstance().getConfig().getUserEncryptTokenHint());
+      }
+      TSFileDescriptor.getInstance().getConfig().setEncryptKeyFromToken(token);
+    }
+    String encryptMagicString = properties.getProperty(ENCRYPT_MAGIC_STRING);
+    byte[] magicStringBytes = EncryptUtils.hexStringToByteArray(encryptMagicString);
+    String decryptedMagicString =
+        new String(
+            EncryptUtils.getEncrypt().getDecryptor().decrypt(magicStringBytes),
+            TSFileConfig.STRING_CHARSET);
+    if (!Objects.equals(decryptedMagicString, magicString)) {
+      logger.error("encrypt_magic_string is not matched");
+      throw new ConfigurationException(
+          "Changing encrypt type or key for tsfile encryption after first start is not permitted. Here is your token hint info: "
+              + CommonDescriptor.getInstance().getConfig().getUserEncryptTokenHint());
     }
   }
 }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java

@@ -43,6 +43,7 @@
 import org.apache.iotdb.rpc.TSStatusCode;
 
 import org.apache.thrift.TException;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -156,6 +157,9 @@ public void run() {
     // Shutdown all consensus pipe's receiver
     PipeDataNodeAgent.receiver().pipeConsensus().closeReceiverExecutor();
 
+    // set encryption key to 16-byte zero.
+    TSFileDescriptor.getInstance().getConfig().setEncryptKey(new byte[16]);
+
     // Actually stop all services started by the DataNode.
     // If we don't call this, services like the RestService are not stopped and I can't re-start
     // it.

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java

@@ -415,6 +415,8 @@ public class CommonConfig {
 
   private volatile Pattern trustedUriPattern = Pattern.compile("file:.*");
 
+  private String userEncryptTokenHint = "not set yet";
+
   private boolean enforceStrongPassword = false;
   private long passwordExpirationDays = -1;
   // an old password cannot be reused within the given interval if >= 0.
@@ -461,6 +463,16 @@ public void setEncryptDecryptProviderParameter(String encryptDecryptProviderPara
     this.encryptDecryptProviderParameter = encryptDecryptProviderParameter;
   }
 
+  public void setUserEncryptTokenHint(String userEncryptTokenHint) {
+    if (userEncryptTokenHint != null && !userEncryptTokenHint.isEmpty()) {
+      this.userEncryptTokenHint = userEncryptTokenHint;
+    }
+  }
+
+  public String getUserEncryptTokenHint() {
+    return userEncryptTokenHint;
+  }
+
   public String getOpenIdProviderUrl() {
     return openIdProviderUrl;
   }

iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java

@@ -90,6 +90,8 @@ public void loadCommonProps(TrimProperties properties) throws IOException {
             "iotdb_server_encrypt_decrypt_provider_parameter",
             config.getEncryptDecryptProviderParameter()));
 
+    config.setUserEncryptTokenHint(System.getenv("user_encrypt_token_hint"));
+
     config.setEnableGrantOption(
         Boolean.parseBoolean(
             properties.getProperty("enable_grant_option", String.valueOf("true"))));