Skip to content

Commit 573be8e

Browse files
leonchen83leonchen83
committed
ignore ttl
1 parent c40abe0 commit 573be8e

8 files changed

Lines changed: 52 additions & 61 deletions

File tree

‎src/main/java/com/moilioncircle/redis/rdb/cli/cmd/XRct.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ public class XRct implements Callable<Integer> {
9898
@Option(names = {"-r", "--replace"}, description = {"Whether the generated aof with <replace>", "parameter(--format dump). if not specified,", "default value is false."})
9999
private boolean replace;
100100

101+
@Option(names = {"-i", "--ignore-ttl"}, description = {"Ignore keys whose TTL is set, default is false."})
102+
private boolean ignoreTTL;
103+
101104
@Override
102105
public Integer call() throws Exception {
103106
source = normalize(source, FileType.RDB, spec, "Invalid options: '--source=<source>'");
@@ -109,7 +112,7 @@ public Integer call() throws Exception {
109112
args.output = output;
110113
args.replace = replace;
111114
args.largest = largest;
112-
args.filter = filter(regexs, db, type);
115+
args.filter = filter(regexs, db, type, ignoreTTL);
113116

114117
Replicator r = new XRedisReplicator(source, configure, DefaultReplFilter.RDB);
115118

‎src/main/java/com/moilioncircle/redis/rdb/cli/cmd/XRdt.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@ public static class Split {
115115
@Option(names = {"-t", "--type"}, arity = "1..*", description = {"Data type to export. possible values are", "string, hash, set, sortedset, list, module, ", "stream. multiple types can be provided. if not", "specified, all data types will be returned."})
116116
private List<String> type;
117117

118+
@Option(names = {"-i", "--ignore-ttl"}, description = {"Ignore keys whose TTL is set, default is false."})
119+
private boolean ignoreTTL;
120+
118121
@Override
119122
public Integer call() throws Exception {
120123
Action action = Action.NONE;
@@ -166,7 +169,7 @@ public Integer call() throws Exception {
166169
arg.config = config;
167170
arg.backup = backup;
168171
arg.output = output;
169-
arg.filter = XFilter.filter(regexs, db, type);
172+
arg.filter = XFilter.filter(regexs, db, type, ignoreTTL);
170173

171174
List<Tuple2<Replicator, String>> list = action.dress(configure, arg);
172175

‎src/main/java/com/moilioncircle/redis/rdb/cli/cmd/XRmt.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -110,17 +110,14 @@ private static class RmtExclusive {
110110
@Option(names = {"-l", "--legacy"}, description = {"If specify the <replace> and this parameter.", "then use lua script to migrate data to target.", "if target redis version is greater than 3.0.", "no need to add this parameter."})
111111
private boolean legacy;
112112

113-
@Option(names = {"-x", "--ignore-keys-with-ttl"}, description = {"Ignore keys whose TTL is set", "default is false."})
114-
private boolean ignoreKeysWithTTL;
113+
@Option(names = {"-i", "--ignore-ttl"}, description = {"Ignore keys whose TTL is set, default is false."})
114+
private boolean ignoreTTL;
115115

116116
@Override
117117
public Integer call() throws Exception {
118118
source = normalize(source, FileType.RDB, spec, "Invalid options: '--source=<source>'");
119119
Configure configure = Configure.bind();
120-
if (ignoreKeysWithTTL) {
121-
configure.setIgnoreKeysWithTTL(true);
122-
}
123-
120+
124121
if (exclusive.migrate != null) {
125122
RedisURI uri = new RedisURI(exclusive.migrate);
126123

@@ -159,7 +156,7 @@ public Integer call() throws Exception {
159156
try (ProgressBar bar = ProgressBar.bar(-1, configure.isEnableProgressBar())) {
160157

161158
Replicator r = new XRedisReplicator(source, configure, DefaultReplFilter.RDB);
162-
r.setRdbVisitor(new ClusterRdbVisitor(r, configure, cluster(regexs, type), null, readAllLines(path), replace));
159+
r.setRdbVisitor(new ClusterRdbVisitor(r, configure, cluster(regexs, type, ignoreTTL), null, readAllLines(path), replace));
163160

164161
r.addEventListener((rep, event) -> {
165162

@@ -184,10 +181,10 @@ private RdbVisitor getRdbVisitor(Replicator replicator, Configure configure, Red
184181
try (XEndpoint endpoint = new XEndpoint(uri.getHost(), uri.getPort(), configure.merge(uri, false))) {
185182
RedisObject r = endpoint.send(CLUSTER, NODES);
186183
if (r.type.isError()) {
187-
return new SingleRdbVisitor(replicator, configure, filter(regexs, db, type), uri, replace, legacy);
184+
return new SingleRdbVisitor(replicator, configure, filter(regexs, db, type, ignoreTTL), uri, replace, legacy);
188185
} else {
189186
List<String> lines = Collections.ofList(r.getString().split("\n"));
190-
return new ClusterRdbVisitor(replicator, configure, cluster(regexs, type), uri, lines, replace);
187+
return new ClusterRdbVisitor(replicator, configure, cluster(regexs, type, ignoreTTL), uri, lines, replace);
191188
}
192189
} catch (Throwable e) {
193190
throw new RuntimeException("failed to connect to " + uri.getHost() + ":" + uri.getPort() + ", reason " + e.getMessage());

‎src/main/java/com/moilioncircle/redis/rdb/cli/conf/Configure.java‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -291,19 +291,6 @@ public Properties properties() {
291291
*/
292292
private int scanStep = 512;
293293

294-
/**
295-
* ignore keys with ttl set
296-
*/
297-
private boolean ignoreKeysWithTTL = false;
298-
299-
public boolean getIgnoreKeysWithTTL() {
300-
return this.ignoreKeysWithTTL;
301-
}
302-
303-
public void setIgnoreKeysWithTTL(boolean choice) {
304-
this.ignoreKeysWithTTL = choice;
305-
}
306-
307294
public int getBatchSize() {
308295
return batchSize;
309296
}

‎src/main/java/com/moilioncircle/redis/rdb/cli/ext/rmt/SingleRdbVisitor.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,6 @@ public class SingleRdbVisitor extends AbstractRmtRdbVisitor implements EventList
6464
private final Configuration conf;
6565
private ThreadLocal<XEndpoint> endpoint = new ThreadLocal<>();
6666

67-
private int totalKeysWithTTL = 0;
68-
6967
//noinspection ThisEscapedInObjectConstruction
7068
public SingleRdbVisitor(Replicator replicator, Configure configure, Filter filter, RedisURI uri, boolean replace, boolean legacy) throws Exception {
7169
super(replicator, configure, filter, replace);
@@ -117,11 +115,6 @@ public void retry(DumpKeyValuePair dkv, int times) {
117115

118116
byte[] expire = ZERO;
119117
if (dkv.getExpiredMs() != null) {
120-
if (configure.getIgnoreKeysWithTTL()) {
121-
totalKeysWithTTL += 1;
122-
return; // ignore keys with ttl
123-
}
124-
125118

126119
long ms = dkv.getExpiredMs() - System.currentTimeMillis();
127120
if (ms <= 0) {

‎src/main/java/com/moilioncircle/redis/rdb/cli/ext/visitor/BaseRdbVisitor.java‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import com.moilioncircle.redis.replicator.rdb.DefaultRdbVisitor;
5353
import com.moilioncircle.redis.replicator.rdb.RdbValueVisitor;
5454
import com.moilioncircle.redis.replicator.rdb.datatype.ContextKeyValuePair;
55+
import com.moilioncircle.redis.replicator.rdb.datatype.ExpiredType;
5556
import com.moilioncircle.redis.replicator.rdb.skip.SkipRdbValueVisitor;
5657
import com.moilioncircle.redis.replicator.util.Strings;
5758

@@ -85,7 +86,7 @@ public Event applyString(RedisInputStream in, int version, ContextKeyValuePair c
8586
try {
8687
BaseRdbParser parser = new BaseRdbParser(in);
8788
byte[] key = parser.rdbLoadEncodedStringObject().first();
88-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STRING, Strings.toString(key));
89+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STRING, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
8990
if (contains) {
9091
if (listener != null) listener.setGuard(DRAIN);
9192
return doApplyString(in, version, key, RDB_TYPE_STRING, context);
@@ -104,7 +105,7 @@ public Event applyList(RedisInputStream in, int version, ContextKeyValuePair con
104105
try {
105106
BaseRdbParser parser = new BaseRdbParser(in);
106107
byte[] key = parser.rdbLoadEncodedStringObject().first();
107-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST, Strings.toString(key));
108+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
108109
if (contains) {
109110
if (listener != null) listener.setGuard(DRAIN);
110111
return doApplyList(in, version, key, RDB_TYPE_LIST, context);
@@ -123,7 +124,7 @@ public Event applySet(RedisInputStream in, int version, ContextKeyValuePair cont
123124
try {
124125
BaseRdbParser parser = new BaseRdbParser(in);
125126
byte[] key = parser.rdbLoadEncodedStringObject().first();
126-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET, Strings.toString(key));
127+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
127128
if (contains) {
128129
if (listener != null) listener.setGuard(DRAIN);
129130
return doApplySet(in, version, key, RDB_TYPE_SET, context);
@@ -142,7 +143,7 @@ public Event applyZSet(RedisInputStream in, int version, ContextKeyValuePair con
142143
try {
143144
BaseRdbParser parser = new BaseRdbParser(in);
144145
byte[] key = parser.rdbLoadEncodedStringObject().first();
145-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET, Strings.toString(key));
146+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
146147
if (contains) {
147148
if (listener != null) listener.setGuard(DRAIN);
148149
return doApplyZSet(in, version, key, RDB_TYPE_ZSET, context);
@@ -161,7 +162,7 @@ public Event applyZSet2(RedisInputStream in, int version, ContextKeyValuePair co
161162
try {
162163
BaseRdbParser parser = new BaseRdbParser(in);
163164
byte[] key = parser.rdbLoadEncodedStringObject().first();
164-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_2, Strings.toString(key));
165+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
165166
if (contains) {
166167
if (listener != null) listener.setGuard(DRAIN);
167168
return doApplyZSet2(in, version, key, RDB_TYPE_ZSET_2, context);
@@ -180,7 +181,7 @@ public Event applyHash(RedisInputStream in, int version, ContextKeyValuePair con
180181
try {
181182
BaseRdbParser parser = new BaseRdbParser(in);
182183
byte[] key = parser.rdbLoadEncodedStringObject().first();
183-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH, Strings.toString(key));
184+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
184185
if (contains) {
185186
if (listener != null) listener.setGuard(DRAIN);
186187
return doApplyHash(in, version, key, RDB_TYPE_HASH, context);
@@ -199,7 +200,7 @@ public Event applyHashZipMap(RedisInputStream in, int version, ContextKeyValuePa
199200
try {
200201
BaseRdbParser parser = new BaseRdbParser(in);
201202
byte[] key = parser.rdbLoadEncodedStringObject().first();
202-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPMAP, Strings.toString(key));
203+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPMAP, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
203204
if (contains) {
204205
if (listener != null) listener.setGuard(DRAIN);
205206
return doApplyHashZipMap(in, version, key, RDB_TYPE_HASH_ZIPMAP, context);
@@ -218,7 +219,7 @@ public Event applyListZipList(RedisInputStream in, int version, ContextKeyValueP
218219
try {
219220
BaseRdbParser parser = new BaseRdbParser(in);
220221
byte[] key = parser.rdbLoadEncodedStringObject().first();
221-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_ZIPLIST, Strings.toString(key));
222+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_ZIPLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
222223
if (contains) {
223224
if (listener != null) listener.setGuard(DRAIN);
224225
return doApplyListZipList(in, version, key, RDB_TYPE_LIST_ZIPLIST, context);
@@ -237,7 +238,7 @@ public Event applySetIntSet(RedisInputStream in, int version, ContextKeyValuePai
237238
try {
238239
BaseRdbParser parser = new BaseRdbParser(in);
239240
byte[] key = parser.rdbLoadEncodedStringObject().first();
240-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET_INTSET, Strings.toString(key));
241+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET_INTSET, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
241242
if (contains) {
242243
if (listener != null) listener.setGuard(DRAIN);
243244
return doApplySetIntSet(in, version, key, RDB_TYPE_SET_INTSET, context);
@@ -256,7 +257,7 @@ public Event applyZSetZipList(RedisInputStream in, int version, ContextKeyValueP
256257
try {
257258
BaseRdbParser parser = new BaseRdbParser(in);
258259
byte[] key = parser.rdbLoadEncodedStringObject().first();
259-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_ZIPLIST, Strings.toString(key));
260+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_ZIPLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
260261
if (contains) {
261262
if (listener != null) listener.setGuard(DRAIN);
262263
return doApplyZSetZipList(in, version, key, RDB_TYPE_ZSET_ZIPLIST, context);
@@ -275,7 +276,7 @@ public Event applyZSetListPack(RedisInputStream in, int version, ContextKeyValue
275276
try {
276277
BaseRdbParser parser = new BaseRdbParser(in);
277278
byte[] key = parser.rdbLoadEncodedStringObject().first();
278-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_LISTPACK, Strings.toString(key));
279+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_LISTPACK, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
279280
if (contains) {
280281
if (listener != null) listener.setGuard(DRAIN);
281282
return doApplyZSetListPack(in, version, key, RDB_TYPE_ZSET_LISTPACK, context);
@@ -294,7 +295,7 @@ public Event applyHashZipList(RedisInputStream in, int version, ContextKeyValueP
294295
try {
295296
BaseRdbParser parser = new BaseRdbParser(in);
296297
byte[] key = parser.rdbLoadEncodedStringObject().first();
297-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPLIST, Strings.toString(key));
298+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
298299
if (contains) {
299300
if (listener != null) listener.setGuard(DRAIN);
300301
return doApplyHashZipList(in, version, key, RDB_TYPE_HASH_ZIPLIST, context);
@@ -313,7 +314,7 @@ public Event applyHashListPack(RedisInputStream in, int version, ContextKeyValue
313314
try {
314315
BaseRdbParser parser = new BaseRdbParser(in);
315316
byte[] key = parser.rdbLoadEncodedStringObject().first();
316-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_LISTPACK, Strings.toString(key));
317+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_LISTPACK, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
317318
if (contains) {
318319
if (listener != null) listener.setGuard(DRAIN);
319320
return doApplyHashListPack(in, version, key, RDB_TYPE_HASH_LISTPACK, context);
@@ -332,7 +333,7 @@ public Event applyListQuickList(RedisInputStream in, int version, ContextKeyValu
332333
try {
333334
BaseRdbParser parser = new BaseRdbParser(in);
334335
byte[] key = parser.rdbLoadEncodedStringObject().first();
335-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST, Strings.toString(key));
336+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
336337
if (contains) {
337338
if (listener != null) listener.setGuard(DRAIN);
338339
return doApplyListQuickList(in, version, key, RDB_TYPE_LIST_QUICKLIST, context);
@@ -351,7 +352,7 @@ public Event applyListQuickList2(RedisInputStream in, int version, ContextKeyVal
351352
try {
352353
BaseRdbParser parser = new BaseRdbParser(in);
353354
byte[] key = parser.rdbLoadEncodedStringObject().first();
354-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST_2, Strings.toString(key));
355+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
355356
if (contains) {
356357
if (listener != null) listener.setGuard(DRAIN);
357358
return doApplyListQuickList2(in, version, key, RDB_TYPE_LIST_QUICKLIST_2, context);
@@ -370,7 +371,7 @@ public Event applyModule(RedisInputStream in, int version, ContextKeyValuePair c
370371
try {
371372
BaseRdbParser parser = new BaseRdbParser(in);
372373
byte[] key = parser.rdbLoadEncodedStringObject().first();
373-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE, Strings.toString(key));
374+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
374375
if (contains) {
375376
if (listener != null) listener.setGuard(DRAIN);
376377
return doApplyModule(in, version, key, RDB_TYPE_MODULE, context);
@@ -389,7 +390,7 @@ public Event applyModule2(RedisInputStream in, int version, ContextKeyValuePair
389390
try {
390391
BaseRdbParser parser = new BaseRdbParser(in);
391392
byte[] key = parser.rdbLoadEncodedStringObject().first();
392-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE_2, Strings.toString(key));
393+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
393394
if (contains) {
394395
if (listener != null) listener.setGuard(DRAIN);
395396
return doApplyModule2(in, version, key, RDB_TYPE_MODULE_2, context);
@@ -408,7 +409,7 @@ public Event applyStreamListPacks(RedisInputStream in, int version, ContextKeyVa
408409
try {
409410
BaseRdbParser parser = new BaseRdbParser(in);
410411
byte[] key = parser.rdbLoadEncodedStringObject().first();
411-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS, Strings.toString(key));
412+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
412413
if (contains) {
413414
if (listener != null) listener.setGuard(DRAIN);
414415
return doApplyStreamListPacks(in, version, key, RDB_TYPE_STREAM_LISTPACKS, context);
@@ -427,7 +428,7 @@ public Event applyStreamListPacks2(RedisInputStream in, int version, ContextKeyV
427428
try {
428429
BaseRdbParser parser = new BaseRdbParser(in);
429430
byte[] key = parser.rdbLoadEncodedStringObject().first();
430-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS_2, Strings.toString(key));
431+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
431432
if (contains) {
432433
if (listener != null) listener.setGuard(DRAIN);
433434
return doApplyStreamListPacks2(in, version, key, RDB_TYPE_STREAM_LISTPACKS_2, context);

‎src/main/java/com/moilioncircle/redis/rdb/cli/filter/Filter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,6 @@ public interface Filter {
2323

2424
boolean contains(long db);
2525

26-
boolean contains(long db, int type, String key);
26+
boolean contains(long db, int type, String key, boolean hasTTL);
2727

2828
}

0 commit comments

Comments
 (0)