Skip to content

Commit 020b219

Browse files
leonchen83leonchen83
committed
Merge branch 'master' into 0.x
2 parents 2834437 + 573be8e commit 020b219

8 files changed

Lines changed: 56 additions & 39 deletions

File tree

‎src/main/java/com/moilioncircle/redis/rdb/cli/cmd/XRct.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ public class XRct implements Callable<Integer> {
9898
@Option(names = {"-r", "--replace"}, description = {"Whether the generated aof with <replace>", "parameter(--format dump). if not specified,", "default value is false."})
9999
private boolean replace;
100100

101+
@Option(names = {"-i", "--ignore-ttl"}, description = {"Ignore keys whose TTL is set, default is false."})
102+
private boolean ignoreTTL;
103+
101104
@Override
102105
public Integer call() throws Exception {
103106
source = normalize(source, FileType.RDB, spec, "Invalid options: '--source=<source>'");
@@ -109,7 +112,7 @@ public Integer call() throws Exception {
109112
args.output = output;
110113
args.replace = replace;
111114
args.largest = largest;
112-
args.filter = filter(regexs, db, type);
115+
args.filter = filter(regexs, db, type, ignoreTTL);
113116

114117
Replicator r = new XRedisReplicator(source, configure, DefaultReplFilter.RDB);
115118

‎src/main/java/com/moilioncircle/redis/rdb/cli/cmd/XRdt.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,9 @@ public static class Split {
115115
@Option(names = {"-t", "--type"}, arity = "1..*", description = {"Data type to export. possible values are", "string, hash, set, sortedset, list, module, ", "stream. multiple types can be provided. if not", "specified, all data types will be returned."})
116116
private List<String> type;
117117

118+
@Option(names = {"-i", "--ignore-ttl"}, description = {"Ignore keys whose TTL is set, default is false."})
119+
private boolean ignoreTTL;
120+
118121
@Override
119122
public Integer call() throws Exception {
120123
Action action = Action.NONE;
@@ -166,7 +169,7 @@ public Integer call() throws Exception {
166169
arg.config = config;
167170
arg.backup = backup;
168171
arg.output = output;
169-
arg.filter = XFilter.filter(regexs, db, type);
172+
arg.filter = XFilter.filter(regexs, db, type, ignoreTTL);
170173

171174
List<Tuple2<Replicator, String>> list = action.dress(configure, arg);
172175

‎src/main/java/com/moilioncircle/redis/rdb/cli/cmd/XRmt.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,10 @@ private static class RmtExclusive {
109109

110110
@Option(names = {"-l", "--legacy"}, description = {"If specify the <replace> and this parameter.", "then use lua script to migrate data to target.", "if target redis version is greater than 3.0.", "no need to add this parameter."})
111111
private boolean legacy;
112-
112+
113+
@Option(names = {"-i", "--ignore-ttl"}, description = {"Ignore keys whose TTL is set, default is false."})
114+
private boolean ignoreTTL;
115+
113116
@Override
114117
public Integer call() throws Exception {
115118
source = normalize(source, FileType.RDB, spec, "Invalid options: '--source=<source>'");
@@ -153,7 +156,7 @@ public Integer call() throws Exception {
153156
try (ProgressBar bar = ProgressBar.bar(-1, configure.isEnableProgressBar())) {
154157

155158
Replicator r = new XRedisReplicator(source, configure, DefaultReplFilter.RDB);
156-
r.setRdbVisitor(new ClusterRdbVisitor(r, configure, cluster(regexs, type), null, readAllLines(path), replace));
159+
r.setRdbVisitor(new ClusterRdbVisitor(r, configure, cluster(regexs, type, ignoreTTL), null, readAllLines(path), replace));
157160

158161
r.addEventListener((rep, event) -> {
159162

@@ -178,10 +181,10 @@ private RdbVisitor getRdbVisitor(Replicator replicator, Configure configure, Red
178181
try (XEndpoint endpoint = new XEndpoint(uri.getHost(), uri.getPort(), configure.merge(uri, false))) {
179182
RedisObject r = endpoint.send(CLUSTER, NODES);
180183
if (r.type.isError()) {
181-
return new SingleRdbVisitor(replicator, configure, filter(regexs, db, type), uri, replace, legacy);
184+
return new SingleRdbVisitor(replicator, configure, filter(regexs, db, type, ignoreTTL), uri, replace, legacy);
182185
} else {
183186
List<String> lines = Collections.ofList(r.getString().split("\n"));
184-
return new ClusterRdbVisitor(replicator, configure, cluster(regexs, type), uri, lines, replace);
187+
return new ClusterRdbVisitor(replicator, configure, cluster(regexs, type, ignoreTTL), uri, lines, replace);
185188
}
186189
} catch (Throwable e) {
187190
throw new RuntimeException("failed to connect to " + uri.getHost() + ":" + uri.getPort() + ", reason " + e.getMessage());

‎src/main/java/com/moilioncircle/redis/rdb/cli/conf/Configure.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ public Properties properties() {
290290
* scan step
291291
*/
292292
private int scanStep = 512;
293-
293+
294294
public int getBatchSize() {
295295
return batchSize;
296296
}

‎src/main/java/com/moilioncircle/redis/rdb/cli/ext/rmt/SingleRdbVisitor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public class SingleRdbVisitor extends AbstractRmtRdbVisitor implements EventList
6363
private volatile byte[] evalSha;
6464
private final Configuration conf;
6565
private ThreadLocal<XEndpoint> endpoint = new ThreadLocal<>();
66-
66+
6767
//noinspection ThisEscapedInObjectConstruction
6868
public SingleRdbVisitor(Replicator replicator, Configure configure, Filter filter, RedisURI uri, boolean replace, boolean legacy) throws Exception {
6969
super(replicator, configure, filter, replace);
@@ -108,14 +108,14 @@ public void retry(DumpKeyValuePair dkv, int times) {
108108
logger.trace("sync rdb event [{}], times {}", new String(dkv.getKey()), times);
109109
try {
110110
DB db = dkv.getDb();
111-
112111
int index;
113112
if (db != null && (index = (int) db.getDbNumber()) != endpoint.get().getDB()) {
114113
endpoint.get().select(true, index);
115114
}
116115

117116
byte[] expire = ZERO;
118117
if (dkv.getExpiredMs() != null) {
118+
119119
long ms = dkv.getExpiredMs() - System.currentTimeMillis();
120120
if (ms <= 0) {
121121
MONITOR.add(ENDPOINT_FAILURE, "expired", 1);

‎src/main/java/com/moilioncircle/redis/rdb/cli/ext/visitor/BaseRdbVisitor.java‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import com.moilioncircle.redis.replicator.rdb.DefaultRdbVisitor;
5555
import com.moilioncircle.redis.replicator.rdb.RdbValueVisitor;
5656
import com.moilioncircle.redis.replicator.rdb.datatype.ContextKeyValuePair;
57+
import com.moilioncircle.redis.replicator.rdb.datatype.ExpiredType;
5758
import com.moilioncircle.redis.replicator.rdb.skip.SkipRdbValueVisitor;
5859
import com.moilioncircle.redis.replicator.util.Strings;
5960

@@ -87,7 +88,7 @@ public Event applyString(RedisInputStream in, int version, ContextKeyValuePair c
8788
try {
8889
BaseRdbParser parser = new BaseRdbParser(in);
8990
byte[] key = parser.rdbLoadEncodedStringObject().first();
90-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STRING, Strings.toString(key));
91+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STRING, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
9192
if (contains) {
9293
if (listener != null) listener.setGuard(DRAIN);
9394
return doApplyString(in, version, key, RDB_TYPE_STRING, context);
@@ -106,7 +107,7 @@ public Event applyList(RedisInputStream in, int version, ContextKeyValuePair con
106107
try {
107108
BaseRdbParser parser = new BaseRdbParser(in);
108109
byte[] key = parser.rdbLoadEncodedStringObject().first();
109-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST, Strings.toString(key));
110+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
110111
if (contains) {
111112
if (listener != null) listener.setGuard(DRAIN);
112113
return doApplyList(in, version, key, RDB_TYPE_LIST, context);
@@ -125,7 +126,7 @@ public Event applySet(RedisInputStream in, int version, ContextKeyValuePair cont
125126
try {
126127
BaseRdbParser parser = new BaseRdbParser(in);
127128
byte[] key = parser.rdbLoadEncodedStringObject().first();
128-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET, Strings.toString(key));
129+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
129130
if (contains) {
130131
if (listener != null) listener.setGuard(DRAIN);
131132
return doApplySet(in, version, key, RDB_TYPE_SET, context);
@@ -163,7 +164,7 @@ public Event applyZSet(RedisInputStream in, int version, ContextKeyValuePair con
163164
try {
164165
BaseRdbParser parser = new BaseRdbParser(in);
165166
byte[] key = parser.rdbLoadEncodedStringObject().first();
166-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET, Strings.toString(key));
167+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
167168
if (contains) {
168169
if (listener != null) listener.setGuard(DRAIN);
169170
return doApplyZSet(in, version, key, RDB_TYPE_ZSET, context);
@@ -182,7 +183,7 @@ public Event applyZSet2(RedisInputStream in, int version, ContextKeyValuePair co
182183
try {
183184
BaseRdbParser parser = new BaseRdbParser(in);
184185
byte[] key = parser.rdbLoadEncodedStringObject().first();
185-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_2, Strings.toString(key));
186+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
186187
if (contains) {
187188
if (listener != null) listener.setGuard(DRAIN);
188189
return doApplyZSet2(in, version, key, RDB_TYPE_ZSET_2, context);
@@ -201,7 +202,7 @@ public Event applyHash(RedisInputStream in, int version, ContextKeyValuePair con
201202
try {
202203
BaseRdbParser parser = new BaseRdbParser(in);
203204
byte[] key = parser.rdbLoadEncodedStringObject().first();
204-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH, Strings.toString(key));
205+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
205206
if (contains) {
206207
if (listener != null) listener.setGuard(DRAIN);
207208
return doApplyHash(in, version, key, RDB_TYPE_HASH, context);
@@ -220,7 +221,7 @@ public Event applyHashZipMap(RedisInputStream in, int version, ContextKeyValuePa
220221
try {
221222
BaseRdbParser parser = new BaseRdbParser(in);
222223
byte[] key = parser.rdbLoadEncodedStringObject().first();
223-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPMAP, Strings.toString(key));
224+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPMAP, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
224225
if (contains) {
225226
if (listener != null) listener.setGuard(DRAIN);
226227
return doApplyHashZipMap(in, version, key, RDB_TYPE_HASH_ZIPMAP, context);
@@ -239,7 +240,7 @@ public Event applyListZipList(RedisInputStream in, int version, ContextKeyValueP
239240
try {
240241
BaseRdbParser parser = new BaseRdbParser(in);
241242
byte[] key = parser.rdbLoadEncodedStringObject().first();
242-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_ZIPLIST, Strings.toString(key));
243+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_ZIPLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
243244
if (contains) {
244245
if (listener != null) listener.setGuard(DRAIN);
245246
return doApplyListZipList(in, version, key, RDB_TYPE_LIST_ZIPLIST, context);
@@ -258,7 +259,7 @@ public Event applySetIntSet(RedisInputStream in, int version, ContextKeyValuePai
258259
try {
259260
BaseRdbParser parser = new BaseRdbParser(in);
260261
byte[] key = parser.rdbLoadEncodedStringObject().first();
261-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET_INTSET, Strings.toString(key));
262+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_SET_INTSET, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
262263
if (contains) {
263264
if (listener != null) listener.setGuard(DRAIN);
264265
return doApplySetIntSet(in, version, key, RDB_TYPE_SET_INTSET, context);
@@ -277,7 +278,7 @@ public Event applyZSetZipList(RedisInputStream in, int version, ContextKeyValueP
277278
try {
278279
BaseRdbParser parser = new BaseRdbParser(in);
279280
byte[] key = parser.rdbLoadEncodedStringObject().first();
280-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_ZIPLIST, Strings.toString(key));
281+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_ZIPLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
281282
if (contains) {
282283
if (listener != null) listener.setGuard(DRAIN);
283284
return doApplyZSetZipList(in, version, key, RDB_TYPE_ZSET_ZIPLIST, context);
@@ -296,7 +297,7 @@ public Event applyZSetListPack(RedisInputStream in, int version, ContextKeyValue
296297
try {
297298
BaseRdbParser parser = new BaseRdbParser(in);
298299
byte[] key = parser.rdbLoadEncodedStringObject().first();
299-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_LISTPACK, Strings.toString(key));
300+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_ZSET_LISTPACK, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
300301
if (contains) {
301302
if (listener != null) listener.setGuard(DRAIN);
302303
return doApplyZSetListPack(in, version, key, RDB_TYPE_ZSET_LISTPACK, context);
@@ -315,7 +316,7 @@ public Event applyHashZipList(RedisInputStream in, int version, ContextKeyValueP
315316
try {
316317
BaseRdbParser parser = new BaseRdbParser(in);
317318
byte[] key = parser.rdbLoadEncodedStringObject().first();
318-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPLIST, Strings.toString(key));
319+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_ZIPLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
319320
if (contains) {
320321
if (listener != null) listener.setGuard(DRAIN);
321322
return doApplyHashZipList(in, version, key, RDB_TYPE_HASH_ZIPLIST, context);
@@ -334,7 +335,7 @@ public Event applyHashListPack(RedisInputStream in, int version, ContextKeyValue
334335
try {
335336
BaseRdbParser parser = new BaseRdbParser(in);
336337
byte[] key = parser.rdbLoadEncodedStringObject().first();
337-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_LISTPACK, Strings.toString(key));
338+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_HASH_LISTPACK, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
338339
if (contains) {
339340
if (listener != null) listener.setGuard(DRAIN);
340341
return doApplyHashListPack(in, version, key, RDB_TYPE_HASH_LISTPACK, context);
@@ -353,7 +354,7 @@ public Event applyListQuickList(RedisInputStream in, int version, ContextKeyValu
353354
try {
354355
BaseRdbParser parser = new BaseRdbParser(in);
355356
byte[] key = parser.rdbLoadEncodedStringObject().first();
356-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST, Strings.toString(key));
357+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
357358
if (contains) {
358359
if (listener != null) listener.setGuard(DRAIN);
359360
return doApplyListQuickList(in, version, key, RDB_TYPE_LIST_QUICKLIST, context);
@@ -372,7 +373,7 @@ public Event applyListQuickList2(RedisInputStream in, int version, ContextKeyVal
372373
try {
373374
BaseRdbParser parser = new BaseRdbParser(in);
374375
byte[] key = parser.rdbLoadEncodedStringObject().first();
375-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST_2, Strings.toString(key));
376+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_LIST_QUICKLIST_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
376377
if (contains) {
377378
if (listener != null) listener.setGuard(DRAIN);
378379
return doApplyListQuickList2(in, version, key, RDB_TYPE_LIST_QUICKLIST_2, context);
@@ -391,7 +392,7 @@ public Event applyModule(RedisInputStream in, int version, ContextKeyValuePair c
391392
try {
392393
BaseRdbParser parser = new BaseRdbParser(in);
393394
byte[] key = parser.rdbLoadEncodedStringObject().first();
394-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE, Strings.toString(key));
395+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
395396
if (contains) {
396397
if (listener != null) listener.setGuard(DRAIN);
397398
return doApplyModule(in, version, key, RDB_TYPE_MODULE, context);
@@ -410,7 +411,7 @@ public Event applyModule2(RedisInputStream in, int version, ContextKeyValuePair
410411
try {
411412
BaseRdbParser parser = new BaseRdbParser(in);
412413
byte[] key = parser.rdbLoadEncodedStringObject().first();
413-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE_2, Strings.toString(key));
414+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_MODULE_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
414415
if (contains) {
415416
if (listener != null) listener.setGuard(DRAIN);
416417
return doApplyModule2(in, version, key, RDB_TYPE_MODULE_2, context);
@@ -429,7 +430,7 @@ public Event applyStreamListPacks(RedisInputStream in, int version, ContextKeyVa
429430
try {
430431
BaseRdbParser parser = new BaseRdbParser(in);
431432
byte[] key = parser.rdbLoadEncodedStringObject().first();
432-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS, Strings.toString(key));
433+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
433434
if (contains) {
434435
if (listener != null) listener.setGuard(DRAIN);
435436
return doApplyStreamListPacks(in, version, key, RDB_TYPE_STREAM_LISTPACKS, context);
@@ -448,7 +449,7 @@ public Event applyStreamListPacks2(RedisInputStream in, int version, ContextKeyV
448449
try {
449450
BaseRdbParser parser = new BaseRdbParser(in);
450451
byte[] key = parser.rdbLoadEncodedStringObject().first();
451-
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS_2, Strings.toString(key));
452+
boolean contains = filter.contains(context.getDb().getDbNumber(), RDB_TYPE_STREAM_LISTPACKS_2, Strings.toString(key), context.getExpiredType() != ExpiredType.NONE);
452453
if (contains) {
453454
if (listener != null) listener.setGuard(DRAIN);
454455
return doApplyStreamListPacks2(in, version, key, RDB_TYPE_STREAM_LISTPACKS_2, context);

‎src/main/java/com/moilioncircle/redis/rdb/cli/filter/Filter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,6 @@ public interface Filter {
2323

2424
boolean contains(long db);
2525

26-
boolean contains(long db, int type, String key);
26+
boolean contains(long db, int type, String key, boolean hasTTL);
2727

2828
}

0 commit comments

Comments
 (0)