Skip to content

Commit 0e2c06c

Browse files
devin-ai-integration[bot]bot_apk
andcommitted
fix(cdk): emit interim heartbeat state for global-cursor substreams with empty parent state
During a long parent walk with mostly-empty children, a global_substream_cursor substream without incremental_dependency emitted no RECORD and no STATE for the whole walk, tripping the Airbyte source heartbeat (86400s). ConcurrentPerPartitionCursor._emit_state_message suppressed the throttled checkpoint STATE whenever the parent state was empty, so the only keepalive signal was withheld exactly when it was needed. Remove the empty-parent guard so the existing 600s-throttled checkpoint STATE is emitted even when parent state is empty. The interim state carries the stable (unadvanced) global cursor, so it is inert on resume (no data loss, no re-read regression). The 600s throttle is unchanged. Co-Authored-By: bot_apk <apk@cognition.ai>
1 parent d828c57 commit 0e2c06c

2 files changed

Lines changed: 46 additions & 4 deletions

File tree

airbyte_cdk/sources/declarative/incremental/concurrent_partition_cursor.py

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -298,9 +298,6 @@ def _emit_state_message(self, throttle: bool = True) -> None:
298298
if current_time is None:
299299
return
300300
self._last_emission_time = current_time
301-
# Skip state emit for global cursor if parent state is empty
302-
if self._use_global_cursor and not self._parent_state:
303-
return
304301

305302
self._connector_state_manager.update_state_for_stream(
306303
self._stream_name,

unit_tests/sources/declarative/incremental/test_concurrent_perpartitioncursor.py

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1021,7 +1021,11 @@ def _run_read(
10211021
"parent_state": {},
10221022
"state": {"created_at": VOTE_100_CREATED_AT},
10231023
},
1024-
1,
1024+
# Two state messages: one interim heartbeat-keepalive checkpoint emitted
1025+
# during the walk (previously suppressed by the empty-parent guard) plus
1026+
# the final state. The interim checkpoint carries the same inert global
1027+
# cursor, so the final state is unchanged.
1028+
2,
10251029
),
10261030
],
10271031
)
@@ -3609,6 +3613,47 @@ def test_state_throttling(mocker):
36093613
mock_repo.emit_message.assert_called_once()
36103614

36113615

3616+
@pytest.mark.parametrize(
3617+
"parent_state",
3618+
[
3619+
pytest.param({}, id="empty_parent_state"),
3620+
pytest.param({"parent": {"updated_at": "2024-01-01"}}, id="non_empty_parent_state"),
3621+
],
3622+
)
3623+
def test_global_cursor_emits_interim_state_even_when_parent_state_empty(mocker, parent_state):
3624+
"""
3625+
Regression test for OC-12977: a global_substream_cursor stream walking a large
3626+
parent with mostly-empty children must still emit its throttled checkpoint STATE
3627+
to keep the Airbyte source heartbeat alive, even when the parent state is empty
3628+
(no ``incremental_dependency``). The previous empty-parent guard suppressed this
3629+
interim emission, causing >24h silence and a heartbeat timeout.
3630+
"""
3631+
cursor = ConcurrentPerPartitionCursor(
3632+
cursor_factory=MagicMock(),
3633+
partition_router=MagicMock(),
3634+
stream_name="test_stream",
3635+
stream_namespace=None,
3636+
stream_state={},
3637+
message_repository=MagicMock(),
3638+
connector_state_manager=MagicMock(),
3639+
connector_state_converter=MagicMock(),
3640+
cursor_field=MagicMock(),
3641+
)
3642+
3643+
cursor._use_global_cursor = True
3644+
cursor._parent_state = parent_state
3645+
cursor._last_emission_time = 0
3646+
3647+
mock_time = mocker.patch("time.time")
3648+
# Exceed the 600s throttle so emission is not throttled.
3649+
mock_time.return_value = 700
3650+
3651+
cursor._emit_state_message()
3652+
3653+
cursor._connector_state_manager.update_state_for_stream.assert_called_once()
3654+
cursor._message_repository.emit_message.assert_called_once()
3655+
3656+
36123657
def test_given_no_partitions_processed_when_close_partition_then_no_state_update():
36133658
mock_cursor = MagicMock()
36143659
# No slices for no partitions

0 commit comments

Comments
 (0)