Skip to content

Commit 07c3bc7

Browse files
authored
CBL-8390: Support comparing and continuing from a non-numeric checkpoint (#2499)
Also updated tools module to release/3.4
1 parent aadf473 commit 07c3bc7

7 files changed

Lines changed: 747 additions & 7 deletions

File tree

LiteCore/tests/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ add_executable(
9090
${TOP}Crypto/CertificateTest.cc
9191
${TOP}Networking/tests/CookieStoreTest.cc
9292
${TOP}Replicator/tests/DBAccessTestWrapper.cc
93+
${TOP}Replicator/tests/ParsedSequenceIDTest.cc
9394
${TOP}Replicator/tests/PropertyEncryptionTests.cc
9495
${TOP}Replicator/tests/ReplicatorLoopbackTest.cc
9596
${TOP}Replicator/tests/ReplicatorAPITest.cc

Replicator/Checkpoint.cc

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -114,19 +114,25 @@ namespace litecore::repl {
114114
if ( _remote && _remote != remoteSequences._remote ) {
115115
LogTo(SyncLog, "Remote sequence mismatch: I had '%s', remote had '%s'", _remote.toJSONString().c_str(),
116116
remoteSequences._remote.toJSONString().c_str());
117-
if ( _remote.isInt() && remoteSequences._remote.isInt() ) {
118-
if ( _remote.intValue() > remoteSequences._remote.intValue() ) {
117+
118+
ParsedSequenceID localParsed, remoteParsed;
119+
bool localParseable = _remote.toParsedSequenceID(localParsed);
120+
bool remoteParseable = remoteSequences._remote.toParsedSequenceID(remoteParsed);
121+
if ( localParseable && remoteParseable ) {
122+
if ( remoteParsed.before(localParsed) ) {
119123
LogTo(SyncLog, "Rolling back to earlier remote sequence from server, some redundant changes may be "
120124
"proposed...");
121125
_remote = remoteSequences._remote;
122126
match = false;
123127
} else {
124-
LogTo(SyncLog, "Ignoring remote sequence on server since client side is older, some redundant "
125-
"changes may be proposed...");
128+
LogTo(SyncLog, "Ignoring remote sequence on server since client side is older or equal, some "
129+
"redundant changes may be proposed...");
126130
}
127131
} else {
128-
Warn("Non-numeric remote sequence detected, resetting replication back to start. Redundant changes "
129-
"will be proposed...");
132+
Warn("Unparseable remote sequence: locally-saved remote seq '%s' is %s, "
133+
"server-side remote seq '%s' is %s. Resetting replication.",
134+
_remote.toJSONString().c_str(), localParseable ? "parseable" : "UNPARSEABLE",
135+
remoteSequences._remote.toJSONString().c_str(), remoteParseable ? "parseable" : "UNPARSEABLE");
130136
_remote = {};
131137
match = false;
132138
}

Replicator/ParsedSequenceID.hh

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
//
2+
// ParsedSequenceID.hh
3+
//
4+
// Copyright 2026-Present Couchbase, Inc.
5+
//
6+
// Use of this software is governed by the Business Source License included
7+
// in the file licenses/BSL-Couchbase.txt. As of the Change Date specified
8+
// in that file, in accordance with the Business Source License, use of this
9+
// software will be governed by the Apache License, Version 2.0, included in
10+
// the file licenses/APL2.txt.
11+
//
12+
13+
#pragma once
14+
#include "StringUtil.hh"
15+
#include "slice_stream.hh"
16+
#include <cstdint>
17+
#include <string>
18+
19+
namespace litecore::repl {
20+
21+
/**
22+
* Represents a parsed Sync Gateway compound sequence ID.
23+
*
24+
* Sync Gateway emits sequence IDs in three formats on the _changes feed:
25+
*
26+
* - Simple: "Seq" e.g. "42"
27+
* A plain database sequence number.
28+
*
29+
* - Backfill: "TriggeredBy:Seq" e.g. "100:35"
30+
* A revision (seq 35) is being sent retroactively because an access-grant
31+
* change at sequence 100 gave the user access to its channel.
32+
*
33+
* - LowSeq: "LowSeq:TriggeredBy:Seq" e.g. "20:100:35"
34+
* Same as backfill, but also records the lowest contiguous sequence (20)
35+
* that the client has fully processed on the feed. TriggeredBy may be
36+
* omitted (e.g. "20::35") when there is no access-grant trigger.
37+
*
38+
* For checkpoint comparison the "comparison value" of each format is:
39+
* - Simple → seq
40+
* - Backfill → triggeredBy
41+
* - LowSeq → lowSeq
42+
*
43+
* The before() method implements the ordering rules that match Sync Gateway's
44+
* SequenceID.Before(), so that LiteCore and SG always agree on which checkpoint is older.
45+
*/
46+
47+
class ParsedSequenceID {
48+
public:
49+
ParsedSequenceID() = default;
50+
51+
ParsedSequenceID(uint64_t seq, uint64_t triggeredBy, uint64_t lowSeq)
52+
: _seq(seq), _triggeredBy(triggeredBy), _lowSeq(lowSeq) {}
53+
54+
[[nodiscard]] uint64_t seq() const { return _seq; }
55+
56+
[[nodiscard]] uint64_t triggeredBy() const { return _triggeredBy; }
57+
58+
[[nodiscard]] uint64_t lowSeq() const { return _lowSeq; }
59+
60+
/// "Seq" — plain sequence, no trigger, no lowSeq.
61+
[[nodiscard]] bool isSimpleRevision() const { return _lowSeq == 0 && _triggeredBy == 0; }
62+
63+
/// "TriggeredBy:Seq" — sent retroactively due to an access-grant, no lowSeq tracking.
64+
/// triggeredBy is the primary comparison value in before().
65+
[[nodiscard]] bool isTriggeredRevision() const { return _lowSeq == 0 && _triggeredBy > 0; }
66+
67+
/// "LowSeq:TriggeredBy:Seq" or "LowSeq::Seq" — includes lowSeq feed-position tracking.
68+
/// lowSeq is the primary comparison value in before(). May or may not include a trigger.
69+
[[nodiscard]] bool isLowSeqRevision() const { return _lowSeq > 0; }
70+
71+
/**
72+
* Parse a sequence string into a ParsedSequenceID.
73+
*
74+
* Accepted formats:
75+
* "42" → {seq=42, triggeredBy=0, lowSeq=0}
76+
* "100:35" → {seq=35, triggeredBy=100, lowSeq=0}
77+
* "20:100:35" → {seq=35, triggeredBy=100, lowSeq=20}
78+
* "20::35" → {seq=35, triggeredBy=0, lowSeq=20}
79+
*
80+
* @param str The string to parse.
81+
* @param out Receives the parsed result on success.
82+
* @return true on success, false if the string is empty, malformed, or has
83+
* an unexpected number of components.
84+
*/
85+
static bool parse(const std::string& str, ParsedSequenceID& out) {
86+
if ( str.empty() || str.back() == ':' ) return false; // litecore::split drops trailing empty tokens
87+
88+
auto parts = litecore::split(str, ":");
89+
if ( parts.size() < 1 || parts.size() > 3 ) return false;
90+
91+
// Parse a single component as uint64. Empty is only valid for middle part of "LowSeq::Seq".
92+
auto readUInt = [](std::string_view sv, uint64_t& val) -> bool {
93+
if ( sv.empty() ) return false;
94+
fleece::slice_istream stream(sv.data(), sv.size());
95+
val = stream.readDecimal();
96+
return stream.eof();
97+
};
98+
99+
uint64_t v[3] = {};
100+
switch ( parts.size() ) {
101+
case 1:
102+
if ( !readUInt(parts[0], v[0]) ) return false;
103+
out = ParsedSequenceID(v[0], 0, 0); // Seq
104+
return true;
105+
case 2:
106+
if ( !readUInt(parts[0], v[0]) || !readUInt(parts[1], v[1]) ) return false;
107+
out = ParsedSequenceID(v[1], v[0], 0); // TriggeredBy:Seq
108+
return true;
109+
case 3:
110+
if ( !readUInt(parts[0], v[0]) ) return false;
111+
if ( !parts[1].empty() && !readUInt(parts[1], v[1]) ) return false; // empty → 0 for "LowSeq::Seq"
112+
if ( !readUInt(parts[2], v[2]) ) return false;
113+
out = ParsedSequenceID(v[2], v[1], v[0]); // LowSeq:TriggeredBy:Seq
114+
return true;
115+
default:
116+
return false;
117+
}
118+
}
119+
120+
/**
121+
* Returns true if this sequence is strictly before \p seqID2.
122+
*
123+
* Implements the same ordering as Sync Gateway's SequenceID.Before().
124+
* Each format has a "comparison value" (CV):
125+
* - Simple → CV = seq
126+
* - Backfill → CV = triggeredBy
127+
* - LowSeq → CV = lowSeq
128+
*
129+
* Cross-format rules (a.before(b)):
130+
*
131+
* a \ b | Simple | Backfill | LowSeq
132+
* ───────────────|──────────────|─────────────────|────────────────
133+
* Simple | seq < seq | seq < trig | seq <= low
134+
* Backfill | trig <= seq | trig < trig | trig <= low
135+
* | | (tie: seq<seq) |
136+
* LowSeq | low < seq | low < trig | low < low
137+
* | | | (tie: recurse
138+
* | | | on inner pair)
139+
*
140+
*/
141+
[[nodiscard]] bool before(const ParsedSequenceID& seqID2) const {
142+
const auto& a = *this;
143+
const auto& b = seqID2;
144+
145+
if ( a.isLowSeqRevision() ) {
146+
if ( b.isLowSeqRevision() ) {
147+
if ( a.lowSeq() != b.lowSeq() ) return a.lowSeq() < b.lowSeq();
148+
ParsedSequenceID aInner(a.seq(), a.triggeredBy(), 0);
149+
ParsedSequenceID bInner(b.seq(), b.triggeredBy(), 0);
150+
return aInner.before(bInner);
151+
}
152+
if ( b.isTriggeredRevision() ) return a.lowSeq() < b.triggeredBy();
153+
/* b is simple seq*/
154+
return a.lowSeq() < b.seq();
155+
}
156+
157+
if ( a.isTriggeredRevision() ) {
158+
if ( b.isLowSeqRevision() ) return a.triggeredBy() <= b.lowSeq();
159+
if ( b.isTriggeredRevision() ) {
160+
if ( a.triggeredBy() != b.triggeredBy() ) return a.triggeredBy() < b.triggeredBy();
161+
return a.seq() < b.seq();
162+
}
163+
/* b is simple */
164+
return a.triggeredBy() <= b.seq();
165+
}
166+
167+
// a is simple revision
168+
if ( b.isLowSeqRevision() ) return a.seq() <= b.lowSeq();
169+
if ( b.isTriggeredRevision() ) return a.seq() < b.triggeredBy();
170+
/* both simple */
171+
return a.seq() < b.seq();
172+
}
173+
174+
private:
175+
uint64_t _seq{0}; ///< The actual internal database sequence number.
176+
uint64_t _triggeredBy{0}; ///< Sequence# of the access-grant that triggered backfill (0 = none).
177+
uint64_t _lowSeq{0}; ///< Lowest contiguous sequence seen on the feed (0 = not present).
178+
};
179+
180+
} // namespace litecore::repl

Replicator/RemoteSequence.hh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//
1212

1313
#pragma once
14+
#include "ParsedSequenceID.hh"
1415
#include "StringUtil.hh"
1516
#include "fleece/Fleece.hh"
1617
#include "slice_stream.hh"
@@ -19,6 +20,7 @@
1920

2021
namespace litecore::repl {
2122

23+
2224
/** A sequence received from a remote peer. Can be any JSON value, but optimized for positive ints. */
2325
class RemoteSequence {
2426
public:
@@ -73,6 +75,21 @@ namespace litecore::repl {
7375

7476
bool operator!=(const RemoteSequence& other) const noexcept FLPURE { return _value != other._value; }
7577

78+
/** Convert this RemoteSequence to a ParsedSequenceID. Returns false if unparseable.
79+
Note: sliceValue() may contain JSON string quotes (added by Value::toJSON() in the
80+
Value constructor path). These are stripped before parsing so that compound Sync
81+
Gateway sequences like "20:100:35" are handled correctly whether quoted or not. */
82+
[[nodiscard]] bool toParsedSequenceID(ParsedSequenceID& out) const {
83+
if ( !*this ) return false;
84+
if ( isInt() ) {
85+
out = {intValue(), 0, 0};
86+
return true;
87+
}
88+
std::string s(sliceValue());
89+
if ( s.size() >= 2 && s.front() == '"' && s.back() == '"' ) s = s.substr(1, s.size() - 2);
90+
return ParsedSequenceID::parse(s, out);
91+
}
92+
7693
bool operator<(const RemoteSequence& other) const noexcept FLPURE {
7794
if ( isInt() ) return !other.isInt() || intValue() < other.intValue();
7895
else

0 commit comments

Comments
 (0)