Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions frontend/src/scenes/session-recordings/playlist/Playlist.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -395,13 +395,42 @@ const TitleWithCount = ({ title, count }: { title?: string; count: number }): JS
)
}

const RecordingsListError = (): JSX.Element => {
const { sessionRecordingsAPIError } = useValues(sessionRecordingsPlaylistLogic)
const { loadSessionRecordings } = useActions(sessionRecordingsPlaylistLogic)

// The chosen filter combination is too heavy for ClickHouse to run — narrowing the filters is the fix.
if (sessionRecordingsAPIError?.code === 'query_too_expensive') {
return (
<LemonBanner type="warning" action={{ children: 'Try again', onClick: () => loadSessionRecordings() }}>
{sessionRecordingsAPIError.detail ||
'This filter combination is too heavy for us to run. Try narrowing your search and search again.'}
</LemonBanner>
)
}

// 429 covers our transient capacity / query-timeout responses — retrying the same search works.
const isTransient = sessionRecordingsAPIError?.status === 429
return (
<LemonBanner
type={isTransient ? 'warning' : 'error'}
action={{ children: 'Try again', onClick: () => loadSessionRecordings() }}
>
{isTransient
? sessionRecordingsAPIError?.detail ||
'Recordings are taking too long to load right now. Please try again in a moment.'
: 'Error while trying to load recordings.'}
</LemonBanner>
)
}

const ListEmptyState = (): JSX.Element => {
const { sessionRecordingsAPIErrored, unusableEventsInFilter } = useValues(sessionRecordingsPlaylistLogic)

return (
<div className="p-3 text-sm text-secondary">
{sessionRecordingsAPIErrored ? (
<LemonBanner type="error">Error while trying to load recordings.</LemonBanner>
<RecordingsListError />
) : unusableEventsInFilter.length ? (
<UnusableEventsWarning unusableEventsInFilter={unusableEventsInFilter} />
) : (
Expand All @@ -425,7 +454,7 @@ const CollectionEmptyState = ({
return (
<div className="p-3 text-sm text-secondary">
{sessionRecordingsAPIErrored ? (
<LemonBanner type="error">Error while trying to load recordings.</LemonBanner>
<RecordingsListError />
) : unusableEventsInFilter.length ? (
<UnusableEventsWarning unusableEventsInFilter={unusableEventsInFilter} />
) : isSynthetic ? (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -755,6 +755,21 @@ export const sessionRecordingsPlaylistLogic = kea<sessionRecordingsPlaylistLogic
loadPrev: () => false,
},
],
sessionRecordingsAPIError: [
null as { detail: string | null; code: string | null; status?: number } | null,
{
loadSessionRecordingsFailure: (_, { errorObject }) => ({
detail: errorObject?.detail ?? null,
code: errorObject?.code ?? null,
status: errorObject?.status,
}),
loadSessionRecordingSuccess: () => null,
setFilters: () => null,
setAdvancedFilters: () => null,
loadNext: () => null,
loadPrev: () => null,
},
],
selectedRecordingsIds: [
[] as string[],
{
Expand Down
11 changes: 11 additions & 0 deletions posthog/api/test/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from posthog.api.utils import (
PaginationMode,
ServerTimingsGathered,
check_definition_ids_inclusion_field_sql,
format_paginated_url,
get_data,
Expand Down Expand Up @@ -338,3 +339,13 @@ def test_unparsed_hostname_in_allowed_url_list(
self, _name: str, allowlist: list[str], needle: str | None, expected: bool
) -> None:
assert unparsed_hostname_in_allowed_url_list(allowlist, needle) == expected

def test_server_timing_header_truncated_below_alb_limit(self) -> None:
# An oversized timings header caused ALBs to return unexplained 502s, so we cap it at 10k.
# Truncation is expected and must not raise (it previously reported a handled exception on every hit).
timer = ServerTimingsGathered()
timer.timings_dict = {f"a_very_long_timing_name_number_{i}": float(i) for i in range(2000)}

header = timer.to_header_string()

assert 0 < len(header) <= 10000
32 changes: 19 additions & 13 deletions posthog/api/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from django.http import HttpRequest

import structlog
from posthoganalytics import capture_exception
from prometheus_client import Counter
from requests.adapters import HTTPAdapter
from rest_framework import request, serializers, status
Expand Down Expand Up @@ -279,6 +278,11 @@ def check_definition_ids_inclusion_field_sql(
"Stray UTF16 surrogates detected and removed from user input.",
)

SERVER_TIMING_HEADER_TRUNCATED_COUNTER = Counter(
"server_timing_header_truncated_total",
"Server-Timing header was truncated to stay under the ALB header size limit.",
)


# keep in sync with posthog/plugin-server/src/utils/db/utils.ts::safeClickhouseString
def safe_clickhouse_string(s: str, with_counter=True) -> str:
Expand Down Expand Up @@ -616,18 +620,20 @@ def to_header_string(self, hogql_timings: list[QueryTiming] | None = None) -> st
new_length = current_length + len(timing_str) + (2 if result else 0)

if new_length > 10000:
"""
The server timings can grow to arbitrary length - in the case that caused us problems over 33,000 characters
AWS ALBs have limits on size for both each individual header and for all headers on a request
If we exceed that limit then the ALB returns a 502 with no other explanation
leading to confusion and distraction
So, we limit here to 10k characters to avoid that issue
The timings header is a debug signal we don't rely on for functionality
so not receiving all timings is not the worse thing in the world
"""
capture_exception(
Exception(f"Server timing header exceeded 10k limit with {len(timings)} timings"),
properties={"generated_so_far": ", ".join(result), "length_of_timings": len(timings)},
# The server timings can grow to arbitrary length - in the case that caused us problems over 33,000 characters
# AWS ALBs have limits on size for both each individual header and for all headers on a request
# If we exceed that limit then the ALB returns a 502 with no other explanation
# leading to confusion and distraction
# So, we limit here to 10k characters to avoid that issue
# The timings header is a debug signal we don't rely on for functionality
# so not receiving all timings is not the worse thing in the world.
# Truncation is expected behaviour, so we count/log it rather than reporting an exception
# (this fires often enough to drown real signals in error tracking).
SERVER_TIMING_HEADER_TRUNCATED_COUNTER.inc()
logger.info(
"server_timing_header_truncated",
number_of_timings=len(timings),
included_so_far=len(result),
)
break

Expand Down
39 changes: 38 additions & 1 deletion posthog/session_recordings/session_recording_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,17 @@
SharingAccessTokenAuthentication,
SharingPasswordProtectedAuthentication,
)
from posthog.clickhouse.client.limit import ConcurrencyLimitExceeded
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.cloud_utils import is_cloud
from posthog.errors import CHQueryErrorCannotScheduleTask, CHQueryErrorTooManySimultaneousQueries
from posthog.event_usage import report_user_action
from posthog.exceptions import (
ClickHouseAtCapacity,
ClickHouseEstimatedQueryExecutionTimeTooLong,
ClickHouseQueryMemoryLimitExceeded,
ClickHouseQueryTimeOut,
)
from posthog.exceptions_capture import capture_exception
from posthog.helpers.impersonation import is_impersonated
from posthog.models import Organization, Team, User
Expand Down Expand Up @@ -181,6 +188,12 @@
labelnames=["location", "auth_type"],
)

SESSION_RECORDING_LIST_QUERY_TOO_EXPENSIVE = Counter(
"session_recording_list_query_too_expensive_total",
"Recordings list queries that exceeded ClickHouse memory/cost limits for the chosen filters",
labelnames=["auth_type"],
)

logger = structlog.get_logger(__name__)
tracer = trace.get_tracer(__name__)

Expand Down Expand Up @@ -916,10 +929,34 @@ def list(self, request: request.Request, *args: Any, **kwargs: Any) -> Response:
if isinstance(e, exceptions.ValidationError):
raise

if isinstance(e, ServerException) and "CHQueryErrorTimeoutExceeded" in str(e):
# Transient capacity/rate-limit failures — retrying the same query later succeeds.
if isinstance(e, (ClickHouseAtCapacity, ConcurrencyLimitExceeded, CHQueryErrorCannotScheduleTask)):
SESSION_RECORDING_THROTTLED.labels(location="clickhouse_at_capacity", auth_type=auth_type).inc()
raise Throttled(detail="Too many simultaneous queries. Try again later.")

if isinstance(e, ClickHouseQueryTimeOut) or (
isinstance(e, ServerException) and "CHQueryErrorTimeoutExceeded" in str(e)
):
SESSION_RECORDING_THROTTLED.labels(location="query_timeout_exceeded", auth_type=auth_type).inc()
raise Throttled(detail="Query timeout exceeded. Try again later.")

# The chosen filter combination is too expensive to run (heavy property/duration filters
# read the events `properties` column and blow ClickHouse's memory ceiling). This is driven
# by user filter choice, not a bug, so we return an actionable, retryable error and skip
# exception reporting to keep this expected failure out of error tracking.
if isinstance(e, (ClickHouseQueryMemoryLimitExceeded, ClickHouseEstimatedQueryExecutionTimeTooLong)) or (
isinstance(e, ServerException) and "MEMORY_LIMIT_EXCEEDED" in str(e)
):
SESSION_RECORDING_LIST_QUERY_TOO_EXPENSIVE.labels(auth_type=auth_type).inc()
return Response(
{
"code": "query_too_expensive",
"detail": "This filter combination is too heavy for us to run. "
"Try narrowing your search — a shorter date range or fewer property filters — and search again.",
},
status=status.HTTP_400_BAD_REQUEST,
)

posthoganalytics.capture_exception(
e,
distinct_id=user_distinct_id,
Expand Down
36 changes: 36 additions & 0 deletions posthog/session_recordings/test/test_session_recordings.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from posthog.schema import LogEntryPropertyFilter, RecordingsQuery

from posthog.errors import CHQueryErrorCannotScheduleTask, CHQueryErrorTooManySimultaneousQueries
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
from posthog.models import Organization, SessionRecording, User
from posthog.models.team import Team
from posthog.models.utils import uuid7
Expand Down Expand Up @@ -1331,11 +1332,26 @@ def test_400_when_invalid_list_query(self) -> None:
CHQueryErrorTooManySimultaneousQueries("Too many simultaneous queries"),
"Too many simultaneous queries. Try again later.",
),
(
"at_capacity",
ClickHouseAtCapacity(),
"Too many simultaneous queries. Try again later.",
),
(
"cannot_schedule_task",
CHQueryErrorCannotScheduleTask("Cannot schedule task", code=439),
"Too many simultaneous queries. Try again later.",
),
(
"timeout_exceeded",
ServerException("CHQueryErrorTimeoutExceeded"),
"Query timeout exceeded. Try again later.",
),
(
"timeout_exceeded_wrapped",
ClickHouseQueryTimeOut(),
"Query timeout exceeded. Try again later.",
),
]
)
@patch("posthog.session_recordings.queries.session_recording_list_from_query.SessionRecordingListFromQuery.run")
Expand All @@ -1350,6 +1366,26 @@ def test_session_recordings_query_errors(self, _name, exception, expected_messag
"type": "throttled_error",
}

@parameterized.expand(
[
("memory_limit_wrapped", ClickHouseQueryMemoryLimitExceeded()),
(
"memory_limit_raw",
ServerException("DB::Exception ... MEMORY_LIMIT_EXCEEDED while reading column properties"),
),
]
)
@patch("posthog.session_recordings.session_recording_api.posthoganalytics.capture_exception")
@patch("posthog.session_recordings.queries.session_recording_list_from_query.SessionRecordingListFromQuery.run")
def test_session_recordings_query_too_expensive(self, _name, exception, mock_run, mock_capture):
# Heavy property/duration filter combinations blow ClickHouse's memory ceiling. This is driven by
# user filter choice, so we return an actionable 400 and must not report it as a bug in error tracking.
mock_run.side_effect = exception
response = self.client.get(f"/api/projects/{self.team.id}/session_recordings")
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert response.json()["code"] == "query_too_expensive"
mock_capture.assert_not_called()

def test_sync_execute_ch_cannot_schedule_task_retry_then_503(self):
"""Test that list_blocks throws CHQueryErrorCannotScheduleTask multiple times and eventually returns 503"""
call_count = 0
Expand Down
Loading