Skip to content

Commit b17b4b5

Browse files
alexluongclaude
andauthored
fix: dragonfly compat issue with lua script in rsmq pkg (#607)
* chore: redis stack compose & testinfra redis setup Cherry-picked from list-tenant branch (9ec3119) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * test: add dragonfly to test compose for compatibility testing Adds Dragonfly service on port 37379 to reproduce issue #573 where RSMQ operations fail with "mismatched message response type: got <nil>" 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * test: refactor rsmq test suite & use dragonfly * test: e2e with dragonfly * chore: gofmt * test: e2e delivery retry * fix: rsmq lua script compat with dragonfly --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 05d8fcb commit b17b4b5

13 files changed

Lines changed: 528 additions & 191 deletions

File tree

.env.test

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# DBs
22
TEST_POSTGRES_URL="localhost:35432"
33
TEST_CLICKHOUSE_URL="localhost:39000"
4+
TEST_REDIS_URL="localhost:36379"
5+
TEST_DRAGONFLY_URL="localhost:37379"
46
# MQs
57
TEST_RABBITMQ_URL="localhost:35672"
68
TEST_LOCALSTACK_URL="localhost:34566"

build/dev/deps/compose.yml

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,16 @@
11
name: "outpost-deps"
22

33
services:
4-
# ============================== Cache ==============================
4+
# ============================== Redis ==============================
55
# LOCAL_DEV_REDIS=1 (default) or LOCAL_DEV_DRAGONFLY=1
66
redis:
77
profiles: ["redis"]
8-
image: redis:7.4-alpine
8+
image: redis/redis-stack-server:latest
99
restart: always
1010
ports:
1111
- "26379:6379"
12-
command: >
13-
redis-server
14-
--save 20 1
15-
--requirepass password
12+
environment:
13+
- REDIS_ARGS=--requirepass password
1614
volumes:
1715
- redis:/data
1816

build/test/compose.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,14 @@ services:
88
dockerfile: ./build/test/Dockerfile.mock
99
ports:
1010
- 35555:5555
11+
redis:
12+
image: redis/redis-stack-server:latest
13+
ports:
14+
- 36379:6379
15+
dragonfly:
16+
image: docker.dragonflydb.io/dragonflydb/dragonfly
17+
ports:
18+
- 37379:6379
1119
clickhouse:
1220
image: clickhouse/clickhouse-server:24-alpine
1321
ports:

cmd/e2e/configs/basic.go

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"log"
77
"os"
8+
"strconv"
89
"strings"
910
"testing"
1011

@@ -104,6 +105,28 @@ func Basic(t *testing.T, opts BasicOpts) config.Config {
104105
return *c
105106
}
106107

108+
// CreateDragonflyConfig returns a Dragonfly config for e2e tests.
109+
// Returns nil if Dragonfly is not configured in testinfra.
110+
func CreateDragonflyConfig(t *testing.T) *redis.RedisConfig {
111+
cfg := testinfra.ReadConfig()
112+
if cfg.DragonflyURL == "" {
113+
return nil
114+
}
115+
116+
parts := strings.Split(cfg.DragonflyURL, ":")
117+
if len(parts) != 2 {
118+
t.Fatalf("Invalid DragonflyURL: %s", cfg.DragonflyURL)
119+
}
120+
port, _ := strconv.Atoi(parts[1])
121+
122+
return &redis.RedisConfig{
123+
Host: parts[0],
124+
Port: port,
125+
Password: "",
126+
Database: 0,
127+
}
128+
}
129+
107130
// CreateRedisClusterConfig creates a Redis cluster configuration for testing
108131
// Returns nil if TEST_REDIS_CLUSTER_URL is not set
109132
func CreateRedisClusterConfig(t *testing.T) *redis.RedisConfig {

cmd/e2e/destwebhook_test.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
"github.com/hookdeck/outpost/cmd/e2e/httpclient"
99
"github.com/hookdeck/outpost/internal/idgen"
10+
"github.com/stretchr/testify/require"
1011
)
1112

1213
// TestingT is an interface wrapper around *testing.T
@@ -1388,3 +1389,104 @@ func (suite *basicSuite) TestDestwebhookFilter() {
13881389
}
13891390
suite.RunAPITests(suite.T(), tests)
13901391
}
1392+
1393+
// TestDeliveryRetry tests that failed deliveries are scheduled for retry via RSMQ.
1394+
// This exercises the RSMQ Lua scripts that are known to fail with Dragonfly.
1395+
func (suite *basicSuite) TestDeliveryRetry() {
1396+
t := suite.T()
1397+
tenantID := idgen.String()
1398+
destinationID := idgen.Destination()
1399+
secret := "testsecret1234567890abcdefghijklmnop"
1400+
1401+
// Setup: create tenant
1402+
resp, err := suite.client.Do(suite.AuthRequest(httpclient.Request{
1403+
Method: httpclient.MethodPUT,
1404+
Path: "/" + tenantID,
1405+
}))
1406+
require.NoError(t, err)
1407+
require.Equal(t, http.StatusCreated, resp.StatusCode)
1408+
1409+
// Setup: configure mock server destination
1410+
resp, err = suite.client.Do(httpclient.Request{
1411+
Method: httpclient.MethodPUT,
1412+
BaseURL: suite.mockServerBaseURL,
1413+
Path: "/destinations",
1414+
Body: map[string]interface{}{
1415+
"id": destinationID,
1416+
"type": "webhook",
1417+
"config": map[string]interface{}{
1418+
"url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID),
1419+
},
1420+
"credentials": map[string]interface{}{
1421+
"secret": secret,
1422+
},
1423+
},
1424+
})
1425+
require.NoError(t, err)
1426+
require.Equal(t, http.StatusOK, resp.StatusCode)
1427+
1428+
// Setup: create destination in outpost
1429+
resp, err = suite.client.Do(suite.AuthRequest(httpclient.Request{
1430+
Method: httpclient.MethodPOST,
1431+
Path: "/" + tenantID + "/destinations",
1432+
Body: map[string]interface{}{
1433+
"id": destinationID,
1434+
"type": "webhook",
1435+
"topics": "*",
1436+
"config": map[string]interface{}{
1437+
"url": fmt.Sprintf("%s/webhook/%s", suite.mockServerBaseURL, destinationID),
1438+
},
1439+
"credentials": map[string]interface{}{
1440+
"secret": secret,
1441+
},
1442+
},
1443+
}))
1444+
require.NoError(t, err)
1445+
require.Equal(t, http.StatusCreated, resp.StatusCode)
1446+
1447+
// Publish event with retry enabled and should_err to force failure
1448+
// This will trigger the RSMQ retry scheduler
1449+
resp, err = suite.client.Do(suite.AuthRequest(httpclient.Request{
1450+
Method: httpclient.MethodPOST,
1451+
Path: "/publish",
1452+
Body: map[string]interface{}{
1453+
"tenant_id": tenantID,
1454+
"topic": "user.created",
1455+
"eligible_for_retry": true, // Enable retry - exercises RSMQ!
1456+
"metadata": map[string]any{
1457+
"should_err": "true", // Force delivery to fail
1458+
},
1459+
"data": map[string]any{
1460+
"test": "retry",
1461+
},
1462+
},
1463+
}))
1464+
require.NoError(t, err)
1465+
require.Equal(t, http.StatusAccepted, resp.StatusCode)
1466+
1467+
// Wait for retry to be scheduled and attempted
1468+
// RetryIntervalSeconds=1 in test config, so 3s should be enough for at least one retry
1469+
time.Sleep(3 * time.Second)
1470+
1471+
// Verify: check that multiple delivery attempts were made (original + retries)
1472+
resp, err = suite.client.Do(httpclient.Request{
1473+
Method: httpclient.MethodGET,
1474+
BaseURL: suite.mockServerBaseURL,
1475+
Path: "/destinations/" + destinationID + "/events",
1476+
})
1477+
require.NoError(t, err)
1478+
require.Equal(t, http.StatusOK, resp.StatusCode)
1479+
1480+
events, ok := resp.Body.([]interface{})
1481+
require.True(t, ok, "response body should be array")
1482+
// Should have more than 1 event if retry worked (original attempt + at least 1 retry)
1483+
require.Greater(t, len(events), 1, "expected multiple delivery attempts (original + retry), got %d", len(events))
1484+
1485+
// Cleanup
1486+
resp, err = suite.client.Do(suite.AuthRequest(httpclient.Request{
1487+
Method: httpclient.MethodDELETE,
1488+
Path: "/" + tenantID,
1489+
}))
1490+
require.NoError(t, err)
1491+
require.Equal(t, http.StatusOK, resp.StatusCode)
1492+
}

cmd/e2e/suites_test.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,24 @@ func TestRedisClusterBasicSuite(t *testing.T) {
212212
})
213213
}
214214

215+
func TestDragonflyBasicSuite(t *testing.T) {
216+
t.Parallel()
217+
if testing.Short() {
218+
t.Skip("skipping e2e test")
219+
}
220+
221+
// Get Dragonfly config from testinfra
222+
dragonflyConfig := configs.CreateDragonflyConfig(t)
223+
if dragonflyConfig == nil {
224+
t.Skip("skipping Dragonfly test (TEST_DRAGONFLY_URL not set)")
225+
}
226+
227+
suite.Run(t, &basicSuite{
228+
logStorageType: configs.LogStorageTypePostgres,
229+
redisConfig: dragonflyConfig,
230+
})
231+
}
232+
215233
func TestBasicSuiteWithDeploymentID(t *testing.T) {
216234
t.Parallel()
217235
if testing.Short() {

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ require (
5050
github.com/testcontainers/testcontainers-go/modules/localstack v0.36.0
5151
github.com/testcontainers/testcontainers-go/modules/postgres v0.36.0
5252
github.com/testcontainers/testcontainers-go/modules/rabbitmq v0.36.0
53+
github.com/testcontainers/testcontainers-go/modules/redis v0.36.0
5354
github.com/uptrace/opentelemetry-go-extra/otelzap v0.3.1
5455
github.com/urfave/cli/v3 v3.4.1
5556
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.53.0

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -906,6 +906,8 @@ github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4
906906
github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
907907
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
908908
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
909+
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
910+
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
909911
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
910912
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
911913
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
@@ -1286,6 +1288,8 @@ github.com/testcontainers/testcontainers-go/modules/postgres v0.36.0 h1:xTGNNsOD
12861288
github.com/testcontainers/testcontainers-go/modules/postgres v0.36.0/go.mod h1:WKS3MGq1lzbVibIRnL08TOaf5bKWPxJe5frzyQfV4oY=
12871289
github.com/testcontainers/testcontainers-go/modules/rabbitmq v0.36.0 h1:gobSVNvTsiJTcGTlVJMpeUfAcz85HAMMwo8xEVQZItE=
12881290
github.com/testcontainers/testcontainers-go/modules/rabbitmq v0.36.0/go.mod h1:rLtFlrLEWcU/Ud52FiGk57QvUqoAHvR380hZo+tkBaI=
1291+
github.com/testcontainers/testcontainers-go/modules/redis v0.36.0 h1:Z+6APQ0DjQP8Kj5Fu+lkAlH2v7f5QkAQyyjnf1Kq8sw=
1292+
github.com/testcontainers/testcontainers-go/modules/redis v0.36.0/go.mod h1:LV66RJhSMikZrxJRc6O0nKcRqykmjQSyX82S93haE2w=
12891293
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
12901294
github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
12911295
github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY=

internal/rsmq/rsmq.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -526,11 +526,12 @@ func (rsmq *RedisSMQ) ReceiveMessage(qname string, vt uint) (*QueueMessage, erro
526526
}
527527

528528
key := rsmq.ns + qname
529+
hashKey := key + q // key + ":Q"
529530

530531
qvt := strconv.FormatUint(queue.ts+uint64(vt)*1000, 10)
531532
ct := strconv.FormatUint(queue.ts, 10)
532533

533-
evalCmd := rsmq.client.EvalSha(hashReceiveMessage, []string{key}, ct, qvt)
534+
evalCmd := rsmq.client.EvalSha(hashReceiveMessage, []string{key, hashKey}, ct, qvt)
534535
return rsmq.createQueueMessage(evalCmd)
535536
}
536537

@@ -546,10 +547,11 @@ func (rsmq *RedisSMQ) PopMessage(qname string) (*QueueMessage, error) {
546547
}
547548

548549
key := rsmq.ns + qname
550+
hashKey := key + q // key + ":Q"
549551

550552
t := strconv.FormatUint(queue.ts, 10)
551553

552-
evalCmd := rsmq.client.EvalSha(hashPopMessage, []string{key}, t)
554+
evalCmd := rsmq.client.EvalSha(hashPopMessage, []string{key, hashKey}, t)
553555
return rsmq.createQueueMessage(evalCmd)
554556
}
555557

0 commit comments

Comments
 (0)