Skip to content

Commit a3de853

Browse files
olamilekan000olamilekan000
committed
feat(dataplane,dashboard): configurable outbound request ID header per project
1 parent d5f9403 commit a3de853

33 files changed

Lines changed: 1415 additions & 873 deletions

‎Makefile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@ ui_install:
1515
build:
1616
scripts/build.sh
1717

18+
.PHONY: ui-build ui-build-fresh
19+
ui-build:
20+
scripts/build-ui.sh
21+
22+
ui-build-fresh:
23+
INSTALL_DEPS=1 scripts/build-ui.sh
24+
1825
.PHONY: test
1926
test:
2027
@go test -race -p 1 $(shell go list ./... | grep -v '/e2e') -v -timeout 30m

‎api/handlers/event.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ func (h *Handler) CreateEndpointEvent(w http.ResponseWriter, r *http.Request) {
5959
}
6060
projectID := project.UID
6161

62+
if err := project.ValidateOutgoingEventIdempotencyKey(newMessage.IdempotencyKey); err != nil {
63+
_ = render.Render(w, r, util.NewErrorResponse(err.Error(), http.StatusBadRequest))
64+
return
65+
}
66+
6267
if !util.IsStringEmpty(newMessage.EndpointID) {
6368
_, err = h.retrieveEndpoint(r.Context(), newMessage.EndpointID, projectID)
6469
if err != nil {

‎api/models/project.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ type ProjectConfig struct {
6969
// Signature is used to configure the project's signature header versions
7070
Signature *SignatureConfiguration `json:"signature"`
7171

72+
// RequestIDHeader is the outbound header name for the stable request id sent on webhook deliveries.
73+
RequestIDHeader config.RequestIDHeaderProvider `json:"request_id_header,omitempty" valid:"optional"`
74+
7275
// MetaEvent is used to configure the project's meta events
7376
MetaEvent *MetaEventConfiguration `json:"meta_event"`
7477

@@ -96,6 +99,7 @@ func (pc *ProjectConfig) Transform() *datastore.ProjectConfig {
9699
RateLimit: pc.RateLimit.Transform(),
97100
Strategy: pc.Strategy.transform(),
98101
Signature: pc.Signature.transform(),
102+
RequestIDHeader: pc.RequestIDHeader,
99103
MetaEvent: pc.MetaEvent.transform(),
100104
CircuitBreaker: pc.CircuitBreaker,
101105
}

‎api/testdb/seed.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ func SeedDefaultProjectWithSSL(db database.Database, orgID string, ssl *datastor
157157
RetryCount: 2,
158158
},
159159
SSL: ssl,
160+
RequestIDHeader: config.DefaultRequestIDHeader,
160161
Signature: &datastore.SignatureConfiguration{
161162
Header: config.DefaultSignatureHeader,
162163
Versions: []datastore.SignatureVersion{

‎api/ui/build/index.html‎

Lines changed: 149 additions & 148 deletions
Large diffs are not rendered by default.

‎config/config.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,7 @@ const (
450450
const (
451451
RedisQueueProvider QueueProvider = "redis"
452452
DefaultSignatureHeader SignatureHeaderProvider = "X-Convoy-Signature"
453+
DefaultRequestIDHeader RequestIDHeaderProvider = "X-Convoy-Idempotency-Key"
453454
PostgresDatabaseProvider DatabaseProvider = "postgres"
454455
)
455456

@@ -471,6 +472,7 @@ type (
471472
AuthProvider string
472473
QueueProvider string
473474
SignatureHeaderProvider string
475+
RequestIDHeaderProvider string
474476
TracerProvider string
475477
CacheProvider string
476478
LimiterProvider string
@@ -483,6 +485,10 @@ func (s SignatureHeaderProvider) String() string {
483485
return string(s)
484486
}
485487

488+
func (r RequestIDHeaderProvider) String() string {
489+
return string(r)
490+
}
491+
486492
type ExecutionMode string
487493

488494
const (

‎datastore/models.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ var (
308308
ReplayAttacks: false,
309309
DisableEndpoint: false,
310310
AddEventIDTraceHeaders: false,
311+
RequestIDHeader: config.DefaultRequestIDHeader,
311312
SSL: &DefaultSSLConfig,
312313
RateLimit: &DefaultRateLimitConfig,
313314
Strategy: &DefaultStrategyConfig,
@@ -626,6 +627,7 @@ type ProjectConfig struct {
626627
RateLimit *RateLimitConfiguration `json:"ratelimit" db:"ratelimit"`
627628
Strategy *StrategyConfiguration `json:"strategy" db:"strategy"`
628629
Signature *SignatureConfiguration `json:"signature" db:"signature"`
630+
RequestIDHeader config.RequestIDHeaderProvider `json:"request_id_header"`
629631
MetaEvent *MetaEventConfiguration `json:"meta_event" db:"meta_event"`
630632
CircuitBreaker *CircuitBreakerConfiguration `json:"circuit_breaker" db:"circuit_breaker"`
631633
}
@@ -651,6 +653,22 @@ func (p *ProjectConfig) GetSignatureConfig() SignatureConfiguration {
651653
return SignatureConfiguration{}
652654
}
653655

656+
func (p *ProjectConfig) GetRequestIDHeader() config.RequestIDHeaderProvider {
657+
if p != nil && strings.TrimSpace(string(p.RequestIDHeader)) != "" {
658+
return p.RequestIDHeader
659+
}
660+
return config.DefaultRequestIDHeader
661+
}
662+
663+
func (p *ProjectConfig) UsesCustomRequestIDHeader() bool {
664+
if p == nil {
665+
return false
666+
}
667+
668+
header := strings.TrimSpace(string(p.RequestIDHeader))
669+
return header != "" && config.RequestIDHeaderProvider(header) != config.DefaultRequestIDHeader
670+
}
671+
654672
func (p *ProjectConfig) GetSSLConfig() SSLConfiguration {
655673
if p.SSL != nil {
656674
return *p.SSL
@@ -759,6 +777,22 @@ func (o *Project) IsDeleted() bool { return o.DeletedAt.Valid }
759777

760778
func (o *Project) IsOwner(e *Endpoint) bool { return o.UID == e.ProjectID }
761779

780+
func (p *Project) ValidateOutgoingEventIdempotencyKey(idempotencyKey string) error {
781+
if p == nil || p.Type != OutgoingProject {
782+
return nil
783+
}
784+
785+
if p.Config == nil || !p.Config.UsesCustomRequestIDHeader() {
786+
return nil
787+
}
788+
789+
if isStringEmpty(idempotencyKey) {
790+
return ErrMissingIdempotencyKeyForCustomRequestIDHeader
791+
}
792+
793+
return nil
794+
}
795+
762796
var (
763797
ErrSignupDisabled = errors.New("user registration is disabled")
764798
ErrUserNotFound = errors.New("user not found")
@@ -778,6 +812,7 @@ var (
778812
ErrNoActiveSecret = errors.New("no active secret found")
779813
ErrSecretNotFound = errors.New("secret not found")
780814
ErrMetaEventNotFound = errors.New("meta event not found")
815+
ErrMissingIdempotencyKeyForCustomRequestIDHeader = errors.New("idempotency_key is required when a custom request_id_header is configured")
781816
)
782817

783818
type AppMetadata struct {

‎datastore/models_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"testing"
55
"time"
66

7+
"github.com/frain-dev/convoy/config"
78
"github.com/stretchr/testify/require"
89
"gopkg.in/guregu/null.v4"
910
)
@@ -44,6 +45,53 @@ func TestProject_IsDeleted(t *testing.T) {
4445
}
4546
}
4647

48+
func TestProject_ValidateOutgoingEventIdempotencyKey(t *testing.T) {
49+
customProject := &Project{
50+
Type: OutgoingProject,
51+
Config: &ProjectConfig{
52+
RequestIDHeader: config.RequestIDHeaderProvider("Split-Request-ID"),
53+
},
54+
}
55+
56+
tests := []struct {
57+
name string
58+
project *Project
59+
idempotencyKey string
60+
wantErr error
61+
}{
62+
{
63+
name: "custom_header_requires_idempotency_key",
64+
project: customProject,
65+
idempotencyKey: "",
66+
wantErr: ErrMissingIdempotencyKeyForCustomRequestIDHeader,
67+
},
68+
{
69+
name: "custom_header_with_idempotency_key",
70+
project: customProject,
71+
idempotencyKey: "stable-request-id",
72+
},
73+
{
74+
name: "default_header_allows_missing_idempotency_key",
75+
project: &Project{Type: OutgoingProject, Config: &ProjectConfig{}},
76+
},
77+
{
78+
name: "incoming_project_allows_missing_idempotency_key",
79+
project: &Project{Type: IncomingProject, Config: &ProjectConfig{RequestIDHeader: "Split-Request-ID"}},
80+
},
81+
}
82+
83+
for _, tc := range tests {
84+
t.Run(tc.name, func(t *testing.T) {
85+
err := tc.project.ValidateOutgoingEventIdempotencyKey(tc.idempotencyKey)
86+
if tc.wantErr != nil {
87+
require.ErrorIs(t, err, tc.wantErr)
88+
return
89+
}
90+
require.NoError(t, err)
91+
})
92+
}
93+
}
94+
4795
func TestProject_IsOwner(t *testing.T) {
4896
tt := []struct {
4997
name string

‎internal/projects/impl.go‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ func projectConfigToCreateParams(id string, config *datastore.ProjectConfig) rep
128128
cb := config.GetCircuitBreakerConfig()
129129
ssl := config.GetSSLConfig()
130130

131+
requestIDHeader := string(config.GetRequestIDHeader())
132+
131133
return repo.CreateProjectConfigurationParams{
132134
ID: common.StringToPgTextNullable(id),
133135
SearchPolicy: common.StringToPgTextNullable(config.SearchPolicy),
@@ -140,6 +142,7 @@ func projectConfigToCreateParams(id string, config *datastore.ProjectConfig) rep
140142
StrategyRetryCount: pgtype.Int4{Int32: int32(sc.RetryCount), Valid: true},
141143
SignatureHeader: common.StringToPgTextNullable(string(sgc.Header)),
142144
SignatureVersions: signatureVersionsToJSON(sgc.Versions),
145+
RequestIDHeader: common.StringToPgTextNullable(requestIDHeader),
143146
DisableEndpoint: pgtype.Bool{Bool: config.DisableEndpoint, Valid: true},
144147
MetaEventsEnabled: pgtype.Bool{Bool: me.IsEnabled, Valid: true},
145148
MetaEventsType: common.StringToPgTextNullable(string(me.Type)),
@@ -163,6 +166,7 @@ func projectConfigToUpdateParams(id string, config *datastore.ProjectConfig) rep
163166
rlc := config.GetRateLimitConfig()
164167
sc := config.GetStrategyConfig()
165168
sgc := config.GetSignatureConfig()
169+
requestIDHeader := string(config.GetRequestIDHeader())
166170
me := config.GetMetaEventConfig()
167171
cb := config.GetCircuitBreakerConfig()
168172
ssl := config.GetSSLConfig()
@@ -178,6 +182,7 @@ func projectConfigToUpdateParams(id string, config *datastore.ProjectConfig) rep
178182
StrategyRetryCount: pgtype.Int4{Int32: int32(sc.RetryCount), Valid: true},
179183
SignatureHeader: common.StringToPgTextNullable(string(sgc.Header)),
180184
SignatureVersions: signatureVersionsToJSON(sgc.Versions),
185+
RequestIDHeader: common.StringToPgTextNullable(requestIDHeader),
181186
DisableEndpoint: pgtype.Bool{Bool: config.DisableEndpoint, Valid: true},
182187
MetaEventsEnabled: pgtype.Bool{Bool: me.IsEnabled, Valid: true},
183188
MetaEventsType: common.StringToPgTextNullable(string(me.Type)),
@@ -205,29 +210,29 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
205210
retainedEvents pgtype.Int4
206211
createdAt, updatedAt, deletedAt pgtype.Timestamptz
207212
// Config fields
208-
searchPolicy pgtype.Text
209-
strategyType, signatureHeader string
210-
signatureVersions []byte
211-
maxPayloadReadSize int32
212-
multipleEndpointSubscriptions bool
213-
replayAttacks bool
214-
ratelimitCount int32
215-
ratelimitDuration int32
216-
strategyDuration int32
217-
strategyRetryCount int32
218-
disableEndpoint bool
219-
sslEnforceSecureEndpoints pgtype.Bool
220-
metaEventsEnabled bool
221-
metaEventsType, metaEventsEventType pgtype.Text
222-
metaEventsUrl, metaEventsSecret pgtype.Text
223-
metaEventsPubSub []byte
224-
cbSampleRate int32
225-
cbErrorTimeout int32
226-
cbFailureThreshold int32
227-
cbSuccessThreshold int32
228-
cbObservabilityWindow int32
229-
cbMinimumRequestCount int32
230-
cbConsecutiveFailureThreshold int32
213+
searchPolicy pgtype.Text
214+
strategyType, signatureHeader, requestIDHeader string
215+
signatureVersions []byte
216+
maxPayloadReadSize int32
217+
multipleEndpointSubscriptions bool
218+
replayAttacks bool
219+
ratelimitCount int32
220+
ratelimitDuration int32
221+
strategyDuration int32
222+
strategyRetryCount int32
223+
disableEndpoint bool
224+
sslEnforceSecureEndpoints pgtype.Bool
225+
metaEventsEnabled bool
226+
metaEventsType, metaEventsEventType pgtype.Text
227+
metaEventsUrl, metaEventsSecret pgtype.Text
228+
metaEventsPubSub []byte
229+
cbSampleRate int32
230+
cbErrorTimeout int32
231+
cbFailureThreshold int32
232+
cbSuccessThreshold int32
233+
cbObservabilityWindow int32
234+
cbMinimumRequestCount int32
235+
cbConsecutiveFailureThreshold int32
231236
)
232237

233238
switch r := row.(type) {
@@ -248,6 +253,7 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
248253
strategyRetryCount = r.ConfigStrategyRetryCount
249254
signatureHeader = r.ConfigSignatureHeader
250255
signatureVersions = r.ConfigSignatureVersions
256+
requestIDHeader = r.ConfigRequestIDHeader
251257
disableEndpoint = r.ConfigDisableEndpoint
252258
sslEnforceSecureEndpoints = r.ConfigSslEnforceSecureEndpoints
253259
metaEventsEnabled = r.ConfigMetaEventsEnabled
@@ -280,6 +286,7 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
280286
strategyRetryCount = r.ConfigStrategyRetryCount
281287
signatureHeader = r.ConfigSignatureHeader
282288
signatureVersions = r.ConfigSignatureVersions
289+
requestIDHeader = r.ConfigRequestIDHeader
283290
disableEndpoint = r.ConfigDisableEndpoint
284291
sslEnforceSecureEndpoints = r.ConfigSslEnforceSecureEndpoints
285292
metaEventsEnabled = r.ConfigMetaEventsEnabled
@@ -333,6 +340,7 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
333340
Header: config.SignatureHeaderProvider(signatureHeader),
334341
Versions: jsonToSignatureVersions(signatureVersions),
335342
},
343+
RequestIDHeader: config.RequestIDHeaderProvider(requestIDHeader),
336344
SSL: &datastore.SSLConfiguration{
337345
EnforceSecureEndpoints: sslEnforceSecureEndpoints.Bool,
338346
},

‎internal/projects/queries.sql‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ INSERT INTO convoy.project_configurations (
55
id, search_policy, max_payload_read_size,
66
replay_attacks_prevention_enabled, ratelimit_count,
77
ratelimit_duration, strategy_type, strategy_duration,
8-
strategy_retry_count, signature_header, signature_versions,
8+
strategy_retry_count, signature_header, signature_versions, request_id_header,
99
disable_endpoint, meta_events_enabled, meta_events_type,
1010
meta_events_event_type, meta_events_url, meta_events_secret,
1111
meta_events_pub_sub, ssl_enforce_secure_endpoints,
@@ -17,7 +17,7 @@ VALUES (
1717
@id, @search_policy, @max_payload_read_size,
1818
@replay_attacks_prevention_enabled, @ratelimit_count,
1919
@ratelimit_duration, @strategy_type, @strategy_duration,
20-
@strategy_retry_count, @signature_header, @signature_versions,
20+
@strategy_retry_count, @signature_header, @signature_versions, @request_id_header,
2121
@disable_endpoint, @meta_events_enabled, @meta_events_type,
2222
@meta_events_event_type, @meta_events_url, @meta_events_secret,
2323
@meta_events_pub_sub, @ssl_enforce_secure_endpoints,
@@ -37,6 +37,7 @@ UPDATE convoy.project_configurations SET
3737
strategy_retry_count = @strategy_retry_count,
3838
signature_header = @signature_header,
3939
signature_versions = @signature_versions,
40+
request_id_header = @request_id_header,
4041
disable_endpoint = @disable_endpoint,
4142
meta_events_enabled = @meta_events_enabled,
4243
meta_events_type = @meta_events_type,
@@ -82,6 +83,7 @@ SELECT
8283
c.strategy_retry_count AS "config_strategy_retry_count",
8384
c.signature_header AS "config_signature_header",
8485
c.signature_versions AS "config_signature_versions",
86+
c.request_id_header AS "config_request_id_header",
8587
c.disable_endpoint AS "config_disable_endpoint",
8688
c.ssl_enforce_secure_endpoints AS "config_ssl_enforce_secure_endpoints",
8789
c.meta_events_enabled AS "config_meta_events_enabled",
@@ -125,6 +127,7 @@ SELECT
125127
c.strategy_retry_count AS "config_strategy_retry_count",
126128
c.signature_header AS "config_signature_header",
127129
c.signature_versions AS "config_signature_versions",
130+
c.request_id_header AS "config_request_id_header",
128131
c.disable_endpoint AS "config_disable_endpoint",
129132
c.ssl_enforce_secure_endpoints AS "config_ssl_enforce_secure_endpoints",
130133
c.meta_events_enabled AS "config_meta_events_enabled",

0 commit comments

Comments
 (0)