Skip to content

Commit ed77464

Browse files
zhujt20zhujt20
committed
encrypt audit log
1 parent 4510242 commit ed77464

49 files changed

Lines changed: 724 additions & 65 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949

5050
import org.apache.tsfile.common.conf.TSFileDescriptor;
5151
import org.apache.tsfile.common.constant.TsFileConstant;
52+
import org.apache.tsfile.encrypt.EncryptParameter;
5253
import org.apache.tsfile.enums.TSDataType;
5354
import org.apache.tsfile.file.metadata.enums.CompressionType;
5455
import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -62,9 +63,11 @@
6263
import java.lang.reflect.Field;
6364
import java.util.ArrayList;
6465
import java.util.Arrays;
66+
import java.util.Collections;
6567
import java.util.List;
6668
import java.util.Objects;
6769
import java.util.Properties;
70+
import java.util.concurrent.ConcurrentHashMap;
6871
import java.util.concurrent.TimeUnit;
6972
import java.util.regex.Pattern;
7073

@@ -1176,6 +1179,10 @@ public class IoTDBConfig {
11761179

11771180
private boolean includeNullValueInWriteThroughputMetric = false;
11781181

1182+
private ConcurrentHashMap<String, EncryptParameter> tsFileDBToEncryptMap =
1183+
new ConcurrentHashMap<>(
1184+
Collections.singletonMap("root.__audit", new EncryptParameter("UNENCRYPTED", null)));
1185+
11791186
IoTDBConfig() {}
11801187

11811188
public int getMaxLogEntriesNumPerBatch() {
@@ -4225,4 +4232,8 @@ public int getPasswordLockTimeMinutes() {
42254232
public void setPasswordLockTimeMinutes(int passwordLockTimeMinutes) {
42264233
this.passwordLockTimeMinutes = passwordLockTimeMinutes;
42274234
}
4235+
4236+
public ConcurrentHashMap<String, EncryptParameter> getTSFileDBToEncryptMap() {
4237+
return tsFileDBToEncryptMap;
4238+
}
42284239
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@
153153
import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool;
154154
import org.apache.iotdb.db.utils.CommonUtils;
155155
import org.apache.iotdb.db.utils.DateTimeUtils;
156+
import org.apache.iotdb.db.utils.EncryptDBUtils;
156157
import org.apache.iotdb.db.utils.ModificationUtils;
157158
import org.apache.iotdb.metrics.utils.MetricLevel;
158159
import org.apache.iotdb.rpc.RpcUtils;
@@ -3087,7 +3088,9 @@ public int executeCompaction() throws InterruptedException {
30873088
if (!isCompactionSelecting.compareAndSet(false, true)) {
30883089
return 0;
30893090
}
3090-
CompactionScheduleContext context = new CompactionScheduleContext();
3091+
CompactionScheduleContext context =
3092+
new CompactionScheduleContext(
3093+
EncryptDBUtils.getFirstEncryptParamFromDatabase(databaseName));
30913094
try {
30923095
List<Long> timePartitions = new ArrayList<>(tsFileManager.getTimePartitions());
30933096
// Sort the time partition from largest to smallest
@@ -3140,7 +3143,9 @@ public int executeTTLCheck() throws InterruptedException {
31403143
return 0;
31413144
}
31423145
logger.info("[TTL] {}-{} Start ttl and modification checking.", databaseName, dataRegionId);
3143-
CompactionScheduleContext context = new CompactionScheduleContext();
3146+
CompactionScheduleContext context =
3147+
new CompactionScheduleContext(
3148+
EncryptDBUtils.getFirstEncryptParamFromDatabase(databaseName));
31443149
List<Long> timePartitions = new ArrayList<>(tsFileManager.getTimePartitions());
31453150
// Sort the time partition from smallest to largest
31463151
Collections.sort(timePartitions);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/CrossCompactionPerformer.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
2525
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
2626

27+
import org.apache.tsfile.encrypt.EncryptParameter;
28+
2729
public enum CrossCompactionPerformer {
2830
READ_POINT,
2931
FAST;
@@ -49,4 +51,16 @@ public ICrossCompactionPerformer createInstance() {
4951
"Illegal compaction performer for cross compaction " + this);
5052
}
5153
}
54+
55+
public ICrossCompactionPerformer createInstance(EncryptParameter encryptParameter) {
56+
switch (this) {
57+
case READ_POINT:
58+
return new ReadPointCompactionPerformer(encryptParameter);
59+
case FAST:
60+
return new FastCompactionPerformer(true, encryptParameter);
61+
default:
62+
throw new IllegalCompactionPerformerException(
63+
"Illegal compaction performer for cross compaction " + this);
64+
}
65+
}
5266
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerSeqCompactionPerformer.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
2525
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
2626

27+
import org.apache.tsfile.encrypt.EncryptParameter;
28+
2729
public enum InnerSeqCompactionPerformer {
2830
READ_CHUNK,
2931
FAST;
@@ -49,4 +51,16 @@ public ISeqCompactionPerformer createInstance() {
4951
"Illegal compaction performer for seq inner compaction " + this);
5052
}
5153
}
54+
55+
public ISeqCompactionPerformer createInstance(EncryptParameter encryptParameter) {
56+
switch (this) {
57+
case READ_CHUNK:
58+
return new ReadChunkCompactionPerformer(encryptParameter);
59+
case FAST:
60+
return new FastCompactionPerformer(false, encryptParameter);
61+
default:
62+
throw new IllegalCompactionPerformerException(
63+
"Illegal compaction performer for seq inner compaction " + this);
64+
}
65+
}
5266
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/constant/InnerUnseqCompactionPerformer.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
2525
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
2626

27+
import org.apache.tsfile.encrypt.EncryptParameter;
28+
2729
public enum InnerUnseqCompactionPerformer {
2830
READ_POINT,
2931
FAST;
@@ -49,4 +51,16 @@ public IUnseqCompactionPerformer createInstance() {
4951
"Illegal compaction performer for unseq inner compaction " + this);
5052
}
5153
}
54+
55+
public IUnseqCompactionPerformer createInstance(EncryptParameter encryptParameter) {
56+
switch (this) {
57+
case READ_POINT:
58+
return new ReadPointCompactionPerformer(encryptParameter);
59+
case FAST:
60+
return new FastCompactionPerformer(false, encryptParameter);
61+
default:
62+
throw new IllegalCompactionPerformerException(
63+
"Illegal compaction performer for unseq inner compaction " + this);
64+
}
65+
}
5266
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java‎

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
4848
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
4949

50+
import org.apache.tsfile.common.conf.TSFileDescriptor;
51+
import org.apache.tsfile.encrypt.EncryptParameter;
5052
import org.apache.tsfile.exception.StopReadTsFileByInterruptException;
5153
import org.apache.tsfile.exception.write.PageException;
5254
import org.apache.tsfile.file.metadata.IDeviceID;
@@ -96,6 +98,8 @@ public class FastCompactionPerformer
9698

9799
private final boolean isCrossCompaction;
98100

101+
private EncryptParameter encryptParameter;
102+
99103
public FastCompactionPerformer(
100104
List<TsFileResource> seqFiles,
101105
List<TsFileResource> unseqFiles,
@@ -109,10 +113,40 @@ public FastCompactionPerformer(
109113
} else {
110114
isCrossCompaction = true;
111115
}
116+
this.encryptParameter =
117+
new EncryptParameter(
118+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
119+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
120+
}
121+
122+
public FastCompactionPerformer(
123+
List<TsFileResource> seqFiles,
124+
List<TsFileResource> unseqFiles,
125+
List<TsFileResource> targetFiles,
126+
EncryptParameter encryptParameter) {
127+
this.seqFiles = seqFiles;
128+
this.unseqFiles = unseqFiles;
129+
this.targetFiles = targetFiles;
130+
if (seqFiles.isEmpty() || unseqFiles.isEmpty()) {
131+
// inner space compaction
132+
isCrossCompaction = false;
133+
} else {
134+
isCrossCompaction = true;
135+
}
136+
this.encryptParameter = encryptParameter;
112137
}
113138

114139
public FastCompactionPerformer(boolean isCrossCompaction) {
115140
this.isCrossCompaction = isCrossCompaction;
141+
this.encryptParameter =
142+
new EncryptParameter(
143+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
144+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
145+
}
146+
147+
public FastCompactionPerformer(boolean isCrossCompaction, EncryptParameter encryptParameter) {
148+
this.isCrossCompaction = isCrossCompaction;
149+
this.encryptParameter = encryptParameter;
116150
}
117151

118152
@Override
@@ -122,8 +156,9 @@ public void perform() throws Exception {
122156
new MultiTsFileDeviceIterator(seqFiles, unseqFiles, readerCacheMap);
123157
AbstractCompactionWriter compactionWriter =
124158
isCrossCompaction
125-
? new FastCrossCompactionWriter(targetFiles, seqFiles, readerCacheMap)
126-
: new FastInnerCompactionWriter(targetFiles)) {
159+
? new FastCrossCompactionWriter(
160+
targetFiles, seqFiles, readerCacheMap, encryptParameter)
161+
: new FastInnerCompactionWriter(targetFiles, encryptParameter)) {
127162
List<Schema> schemas =
128163
CompactionTableSchemaCollector.collectSchema(seqFiles, unseqFiles, readerCacheMap);
129164
compactionWriter.setSchemaForAllTargetFile(schemas);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
3737
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
3838

39+
import org.apache.tsfile.common.conf.TSFileDescriptor;
40+
import org.apache.tsfile.encrypt.EncryptParameter;
3941
import org.apache.tsfile.enums.TSDataType;
4042
import org.apache.tsfile.exception.write.PageException;
4143
import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata;
@@ -66,6 +68,8 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer {
6668
* IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
6769
private Schema schema = null;
6870

71+
private EncryptParameter firstEncryptParameter;
72+
6973
public ReadChunkCompactionPerformer(List<TsFileResource> sourceFiles, TsFileResource targetFile) {
7074
this(sourceFiles, Collections.singletonList(targetFile));
7175
}
@@ -74,13 +78,30 @@ public ReadChunkCompactionPerformer(
7478
List<TsFileResource> sourceFiles, List<TsFileResource> targetFiles) {
7579
setSourceFiles(sourceFiles);
7680
setTargetFiles(targetFiles);
81+
this.firstEncryptParameter =
82+
new EncryptParameter(
83+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
84+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
7785
}
7886

7987
public ReadChunkCompactionPerformer(List<TsFileResource> sourceFiles) {
8088
setSourceFiles(sourceFiles);
89+
this.firstEncryptParameter =
90+
new EncryptParameter(
91+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
92+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
93+
}
94+
95+
public ReadChunkCompactionPerformer() {
96+
this.firstEncryptParameter =
97+
new EncryptParameter(
98+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
99+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
81100
}
82101

83-
public ReadChunkCompactionPerformer() {}
102+
public ReadChunkCompactionPerformer(EncryptParameter encryptParameter) {
103+
this.firstEncryptParameter = encryptParameter;
104+
}
84105

85106
@Override
86107
public void perform()
@@ -161,7 +182,8 @@ private void useNewWriter() throws IOException {
161182
new CompactionTsFileWriter(
162183
targetResources.get(currentTargetFileIndex).getTsFile(),
163184
memoryBudgetForFileWriter,
164-
CompactionType.INNER_SEQ_COMPACTION);
185+
CompactionType.INNER_SEQ_COMPACTION,
186+
firstEncryptParameter);
165187
currentWriter.setSchema(CompactionTableSchemaCollector.copySchema(schema));
166188
}
167189

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,10 @@
4444
import org.apache.iotdb.db.storageengine.dataregion.read.control.QueryResourceManager;
4545
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
4646

47+
import org.apache.tsfile.common.conf.TSFileDescriptor;
4748
import org.apache.tsfile.common.constant.TsFileConstant;
49+
import org.apache.tsfile.encrypt.EncryptParameter;
50+
import org.apache.tsfile.encrypt.EncryptUtils;
4851
import org.apache.tsfile.file.metadata.IDeviceID;
4952
import org.apache.tsfile.read.common.block.TsBlock;
5053
import org.apache.tsfile.read.reader.IPointReader;
@@ -83,22 +86,61 @@ public class ReadPointCompactionPerformer
8386

8487
protected List<TsFileResource> targetFiles = Collections.emptyList();
8588

89+
private EncryptParameter encryptParameter;
90+
8691
public ReadPointCompactionPerformer(
8792
List<TsFileResource> seqFiles,
8893
List<TsFileResource> unseqFiles,
8994
List<TsFileResource> targetFiles) {
9095
this.seqFiles = seqFiles;
9196
this.unseqFiles = unseqFiles;
9297
this.targetFiles = targetFiles;
98+
this.encryptParameter =
99+
new EncryptParameter(
100+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
101+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
102+
}
103+
104+
public ReadPointCompactionPerformer(
105+
List<TsFileResource> seqFiles,
106+
List<TsFileResource> unseqFiles,
107+
List<TsFileResource> targetFiles,
108+
EncryptParameter encryptParameter) {
109+
this.seqFiles = seqFiles;
110+
this.unseqFiles = unseqFiles;
111+
this.targetFiles = targetFiles;
112+
this.encryptParameter = encryptParameter;
93113
}
94114

95115
public ReadPointCompactionPerformer(
96116
List<TsFileResource> seqFiles, List<TsFileResource> unseqFiles) {
97117
this.seqFiles = seqFiles;
98118
this.unseqFiles = unseqFiles;
119+
this.encryptParameter =
120+
new EncryptParameter(
121+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
122+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
99123
}
100124

101-
public ReadPointCompactionPerformer() {}
125+
public ReadPointCompactionPerformer(
126+
List<TsFileResource> seqFiles,
127+
List<TsFileResource> unseqFiles,
128+
EncryptParameter encryptParameter) {
129+
this.seqFiles = seqFiles;
130+
this.unseqFiles = unseqFiles;
131+
this.encryptParameter = encryptParameter;
132+
}
133+
134+
public ReadPointCompactionPerformer() {
135+
this.encryptParameter =
136+
new EncryptParameter(
137+
TSFileDescriptor.getInstance().getConfig().getEncryptType(),
138+
TSFileDescriptor.getInstance().getConfig().getEncryptKey());
139+
}
140+
141+
public ReadPointCompactionPerformer(EncryptParameter encryptParameter) {
142+
this.encryptParameter = encryptParameter;
143+
}
102144

103145
@SuppressWarnings("squid:S2095") // Do not close device iterator
104146
@Override
@@ -159,6 +201,10 @@ public void setSummary(CompactionTaskSummary summary) {
159201
this.summary = summary;
160202
}
161203

204+
public EncryptParameter getEncryptParameter() {
205+
return encryptParameter;
206+
}
207+
162208
private void compactAlignedSeries(
163209
IDeviceID device,
164210
MultiTsFileDeviceIterator deviceIterator,
@@ -194,7 +240,10 @@ private void compactAlignedSeries(
194240
measurementSchemas.add(0, timeSchema);
195241
compactionWriter.startMeasurement(
196242
TsFileConstant.TIME_COLUMN_ID,
197-
new AlignedChunkWriterImpl(measurementSchemas.remove(0), measurementSchemas),
243+
new AlignedChunkWriterImpl(
244+
measurementSchemas.remove(0),
245+
measurementSchemas,
246+
EncryptUtils.getEncryptParameter(getEncryptParameter())),
198247
0);
199248
writeWithReader(compactionWriter, dataBlockReader, device, 0, true);
200249
compactionWriter.endMeasurement(0);
@@ -303,10 +352,11 @@ protected AbstractCompactionWriter getCompactionWriter(
303352
throws IOException {
304353
if (!seqFileResources.isEmpty() && !unseqFileResources.isEmpty()) {
305354
// cross space
306-
return new ReadPointCrossCompactionWriter(targetFileResources, seqFileResources);
355+
return new ReadPointCrossCompactionWriter(
356+
targetFileResources, seqFileResources, encryptParameter);
307357
} else {
308358
// inner space
309-
return new ReadPointInnerCompactionWriter(targetFileResources);
359+
return new ReadPointInnerCompactionWriter(targetFileResources, encryptParameter);
310360
}
311361
}
312362

0 commit comments

Comments
 (0)