Skip to content

Commit 4e2c7ae

Browse files
sentry-junior[bot]Junior (Sentry Bot)grichacodex
authored
fix(config): Migrate PyMemcacheCache to ReconnectingMemcache (#4338)
* fix(config): Swap PyMemcacheCache to ReconnectingMemcache in example config PyMemcacheCache is not thread-safe when used with ContextPropagatingThreadPoolExecutor, which was introduced in getsentry/sentry#111568 (shipped in 26.4.0). Multiple worker threads share a single pymemcache HashClient via copied contextvars, causing protocol corruption and _recv() deadlocks in ingest-monitors and ingest-occurrences consumers. The sentry image was already fixed in getsentry/sentry#113871 (swapped self-hosted/sentry.conf.py to ReconnectingMemcache), but the self-hosted repo's example config still used PyMemcacheCache. Since docker-compose bind-mounts ./sentry:/etc/sentry, the user's config (copied from this example) overrides the image's bundled config. Changes: - sentry.conf.example.py: swap BACKEND to ReconnectingMemcache, add reconnect_age option - check-memcached-backend.sh: accept ReconnectingMemcache as the happy path, error on PyMemcacheCache with migration instructions Fixes #4301 * fix: auto-migrate PyMemcacheCache → ReconnectingMemcache in install.sh Instead of erroring when PyMemcacheCache is found, follow the established pattern from migrate-pgbouncer.sh: prompt the user (or respect APPLY_AUTOMATIC_CONFIG_UPDATES) and sed the config in-place. The migration: 1. Swaps the BACKEND string 2. Adds reconnect_age to OPTIONS if missing * fix: Harden memcached backend migration Handle custom PyMemcacheCache option dictionaries by appending a small fallback that sets reconnect_age when the stock one-line OPTIONS rewrite does not match. Add focused unit coverage for current, stock, custom, opt-out, and legacy backend cases. Fixes #4301 Co-Authored-By: OpenAI Codex <codex@openai.com> * fix(config): Stop install when memcache migration skipped Exit when PyMemcacheCache migration is declined, explicitly disabled, or cannot read prompt input. This prevents install.sh from continuing with a memcache backend known to deadlock ingest workers. Refs GH-4338 Co-Authored-By: Codex <codex@openai.com> --------- Co-authored-by: Junior (Sentry Bot) <junior[bot]@sentry.io> Co-authored-by: Greg Pstrucha <875316+gricha@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
1 parent 7fa000f commit 4e2c7ae

3 files changed

Lines changed: 225 additions & 10 deletions

File tree

‎_unit-test/check-memcached-backend-test.sh‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/usr/bin/env bash
2+
3+
source _unit-test/_test_setup.sh
4+
source install/ensure-files-from-examples.sh
5+
6+
PYMEMCACHE_BACKEND="django.core.cache.backends.memcached.PyMemcacheCache"
7+
RECONNECTING_MEMCACHE_BACKEND="sentry.cache.backends.reconnectingmemcache.ReconnectingMemcache"
8+
MEMCACHED_BACKEND="django.core.cache.backends.memcached.MemcachedCache"
9+
10+
assert_contains() {
11+
local file="$1"
12+
local expected="$2"
13+
14+
if ! grep -Fq "$expected" "$file"; then
15+
echo "Expected $file to contain:"
16+
echo "$expected"
17+
echo "Actual:"
18+
cat "$file"
19+
exit 1
20+
fi
21+
}
22+
23+
assert_not_contains() {
24+
local file="$1"
25+
local unexpected="$2"
26+
27+
if grep -Fq "$unexpected" "$file"; then
28+
echo "Expected $file not to contain:"
29+
echo "$unexpected"
30+
echo "Actual:"
31+
cat "$file"
32+
exit 1
33+
fi
34+
}
35+
36+
write_stock_pymemcache_config() {
37+
cat >"$SENTRY_CONFIG_PY" <<EOF
38+
CACHES = {
39+
"default": {
40+
"BACKEND": "$PYMEMCACHE_BACKEND",
41+
"LOCATION": ["memcached:11211"],
42+
"TIMEOUT": 3600,
43+
"OPTIONS": {"ignore_exc": True},
44+
}
45+
}
46+
EOF
47+
}
48+
49+
write_custom_pymemcache_config() {
50+
cat >"$SENTRY_CONFIG_PY" <<EOF
51+
CACHES = {
52+
"default": {
53+
"BACKEND": "$PYMEMCACHE_BACKEND",
54+
"LOCATION": ["memcached:11211"],
55+
"TIMEOUT": 3600,
56+
"OPTIONS": {"ignore_exc": True, "timeout": 5, "connect_timeout": 3},
57+
}
58+
}
59+
EOF
60+
}
61+
62+
write_memcached_config() {
63+
cat >"$SENTRY_CONFIG_PY" <<EOF
64+
CACHES = {
65+
"default": {
66+
"BACKEND": "$MEMCACHED_BACKEND",
67+
"LOCATION": ["memcached:11211"],
68+
"TIMEOUT": 3600,
69+
"OPTIONS": {"ignore_exc": True},
70+
}
71+
}
72+
EOF
73+
}
74+
75+
echo "Test 1 (current example config)"
76+
export APPLY_AUTOMATIC_CONFIG_UPDATES=1
77+
source install/check-memcached-backend.sh
78+
assert_contains "$SENTRY_CONFIG_PY" "$RECONNECTING_MEMCACHE_BACKEND"
79+
assert_contains "$SENTRY_CONFIG_PY" '"reconnect_age": 300'
80+
81+
echo "Test 2 (stock PyMemcacheCache config)"
82+
write_stock_pymemcache_config
83+
source install/check-memcached-backend.sh
84+
assert_contains "$SENTRY_CONFIG_PY" "$RECONNECTING_MEMCACHE_BACKEND"
85+
assert_contains "$SENTRY_CONFIG_PY" '"OPTIONS": {"ignore_exc": True, "reconnect_age": 300}'
86+
assert_not_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
87+
88+
echo "Test 3 (custom PyMemcacheCache options)"
89+
write_custom_pymemcache_config
90+
source install/check-memcached-backend.sh
91+
assert_contains "$SENTRY_CONFIG_PY" "$RECONNECTING_MEMCACHE_BACKEND"
92+
assert_contains "$SENTRY_CONFIG_PY" 'setdefault("reconnect_age", 300)'
93+
assert_contains "$SENTRY_CONFIG_PY" 'CACHES["default"].get("OPTIONS") is None'
94+
assert_contains "$SENTRY_CONFIG_PY" '"timeout": 5'
95+
assert_not_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
96+
97+
echo "Test 4 (PyMemcacheCache with automatic updates disabled)"
98+
write_stock_pymemcache_config
99+
if (APPLY_AUTOMATIC_CONFIG_UPDATES=0 source install/check-memcached-backend.sh); then
100+
echo "Expected check-memcached-backend.sh to fail when PyMemcacheCache is not migrated"
101+
exit 1
102+
fi
103+
assert_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
104+
105+
echo "Test 5 (PyMemcacheCache with automatic updates declined)"
106+
write_stock_pymemcache_config
107+
if (
108+
unset APPLY_AUTOMATIC_CONFIG_UPDATES
109+
printf 'n\n' | source install/check-memcached-backend.sh
110+
); then
111+
echo "Expected check-memcached-backend.sh to fail when PyMemcacheCache migration is declined"
112+
exit 1
113+
fi
114+
assert_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
115+
116+
echo "Test 6 (PyMemcacheCache with prompt input unavailable)"
117+
write_stock_pymemcache_config
118+
if (
119+
unset APPLY_AUTOMATIC_CONFIG_UPDATES
120+
source install/check-memcached-backend.sh </dev/null
121+
); then
122+
echo "Expected check-memcached-backend.sh to fail when prompt input is unavailable"
123+
exit 1
124+
fi
125+
assert_contains "$SENTRY_CONFIG_PY" "$PYMEMCACHE_BACKEND"
126+
127+
echo "Test 7 (legacy MemcachedCache config)"
128+
write_memcached_config
129+
if (APPLY_AUTOMATIC_CONFIG_UPDATES=1 source install/check-memcached-backend.sh); then
130+
echo "Expected check-memcached-backend.sh to fail on legacy MemcachedCache"
131+
exit 1
132+
fi
133+
134+
report_success

‎install/check-memcached-backend.sh‎

Lines changed: 89 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,97 @@
11
echo "${_group}Checking memcached backend ..."
22

3-
if grep -q "\.PyMemcacheCache" "$SENTRY_CONFIG_PY"; then
4-
echo "PyMemcacheCache found in $SENTRY_CONFIG_PY, gonna assume you're good."
5-
else
6-
if grep -q "\.MemcachedCache" "$SENTRY_CONFIG_PY"; then
7-
echo "MemcachedCache found in $SENTRY_CONFIG_PY, you should switch to PyMemcacheCache."
3+
if grep -q "\.ReconnectingMemcache" "$SENTRY_CONFIG_PY"; then
4+
echo "ReconnectingMemcache found in $SENTRY_CONFIG_PY, you're good."
5+
elif grep -q "\.PyMemcacheCache" "$SENTRY_CONFIG_PY"; then
6+
# PyMemcacheCache is not thread-safe under ContextPropagatingThreadPoolExecutor
7+
# and causes ingest-monitors / ingest-occurrences to deadlock.
8+
# See: https://github.com/getsentry/self-hosted/issues/4301
9+
10+
apply_config_changes_memcache=0
11+
if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then
12+
echo
13+
echo "PyMemcacheCache is no longer safe to use. The monitor consumer now runs"
14+
echo "check-in processing in a ContextPropagatingThreadPoolExecutor, which shares"
15+
echo "the non-thread-safe pymemcache client across threads, causing deadlocks."
16+
echo
17+
echo "We need to swap to ReconnectingMemcache (per-thread clients) in your config."
18+
echo "Do you want us to make this change automatically for you?"
19+
echo
20+
21+
yn=""
22+
until [ ! -z "$yn" ]; do
23+
if ! read -p "y or n? " yn; then
24+
echo
25+
echo "Unable to read a response, so install cannot continue safely."
26+
echo "Update sentry.conf.py manually or rerun with --apply-automatic-config-updates."
27+
echo "See: https://github.com/getsentry/self-hosted/issues/4301"
28+
exit 1
29+
fi
30+
case $yn in
31+
y | yes | 1)
32+
export apply_config_changes_memcache=1
33+
echo
34+
echo -n "Thank you."
35+
;;
36+
n | no | 0)
37+
export apply_config_changes_memcache=0
38+
echo
39+
echo -n "Alright, you will need to update your sentry.conf.py file manually."
40+
echo " See: https://github.com/getsentry/self-hosted/issues/4301"
41+
exit 1
42+
;;
43+
*) yn="" ;;
44+
esac
45+
done
46+
47+
echo
48+
echo "To avoid this prompt in the future, use one of these flags:"
49+
echo
50+
echo " --apply-automatic-config-updates"
51+
echo " --no-apply-automatic-config-updates"
52+
echo
53+
echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:"
54+
echo
55+
echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates"
56+
echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates"
57+
echo
58+
sleep 5
59+
fi
60+
61+
if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_memcache" == 1 ]]; then
62+
echo "Migrating $SENTRY_CONFIG_PY to use ReconnectingMemcache"
63+
sed -i 's|django\.core\.cache\.backends\.memcached\.PyMemcacheCache|sentry.cache.backends.reconnectingmemcache.ReconnectingMemcache|g' "$SENTRY_CONFIG_PY"
64+
65+
if ! grep -q "reconnect_age" "$SENTRY_CONFIG_PY"; then
66+
sed -i 's/"ignore_exc": True}/"ignore_exc": True, "reconnect_age": 300}/g' "$SENTRY_CONFIG_PY"
67+
fi
68+
69+
if ! grep -q "reconnect_age" "$SENTRY_CONFIG_PY"; then
70+
cat <<'EOF' >>"$SENTRY_CONFIG_PY"
71+
72+
# Added by self-hosted install to keep ReconnectingMemcache aligned with
73+
# the bundled Sentry image configuration.
74+
if "CACHES" in globals() and "default" in CACHES:
75+
if CACHES["default"].get("OPTIONS") is None:
76+
CACHES["default"]["OPTIONS"] = {}
77+
CACHES["default"]["OPTIONS"].setdefault("reconnect_age", 300)
78+
EOF
79+
fi
80+
81+
echo "Migrated $SENTRY_CONFIG_PY to use ReconnectingMemcache"
82+
else
83+
echo "PyMemcacheCache found in $SENTRY_CONFIG_PY, install cannot continue until you switch to ReconnectingMemcache."
884
echo "See:"
9-
echo " https://develop.sentry.dev/self-hosted/releases/#breaking-changes"
85+
echo " https://github.com/getsentry/self-hosted/issues/4301"
1086
exit 1
11-
else
12-
echo 'Your setup looks weird. Good luck.'
1387
fi
88+
elif grep -q "\.MemcachedCache" "$SENTRY_CONFIG_PY"; then
89+
echo "MemcachedCache found in $SENTRY_CONFIG_PY, you should switch to ReconnectingMemcache."
90+
echo "See:"
91+
echo " https://develop.sentry.dev/self-hosted/releases/#breaking-changes"
92+
exit 1
93+
else
94+
echo 'Your setup looks weird. Good luck.'
1495
fi
1596

1697
echo "${_endgroup}"

‎sentry/sentry.conf.example.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -208,10 +208,10 @@ def get_internal_network():
208208

209209
CACHES = {
210210
"default": {
211-
"BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache",
211+
"BACKEND": "sentry.cache.backends.reconnectingmemcache.ReconnectingMemcache",
212212
"LOCATION": ["memcached:11211"],
213213
"TIMEOUT": 3600,
214-
"OPTIONS": {"ignore_exc": True},
214+
"OPTIONS": {"ignore_exc": True, "reconnect_age": 300},
215215
}
216216
}
217217

0 commit comments

Comments
 (0)