Skip to content

feat(dataplane,dashboard): configurable outbound request ID header per project #2665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
olamilekan000 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(dataplane,dashboard): configurable outbound request ID header per project #2665

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@ ui_install:
build:
scripts/build.sh

.PHONY: ui-build ui-build-fresh
ui-build:
scripts/build-ui.sh

ui-build-fresh:
INSTALL_DEPS=1 scripts/build-ui.sh

.PHONY: test
test:
@go test -race -p 1 $(shell go list ./... | grep -v '/e2e') -v -timeout 30m
Expand Down
5 changes: 5 additions & 0 deletions api/handlers/event.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ func (h *Handler) CreateEndpointEvent(w http.ResponseWriter, r *http.Request) {
}
projectID := project.UID

if err := project.ValidateOutgoingEventIdempotencyKey(newMessage.IdempotencyKey); err != nil {
_ = render.Render(w, r, util.NewErrorResponse(err.Error(), http.StatusBadRequest))
return
}

if !util.IsStringEmpty(newMessage.EndpointID) {
_, err = h.retrieveEndpoint(r.Context(), newMessage.EndpointID, projectID)
if err != nil {
Expand Down
4 changes: 4 additions & 0 deletions api/models/project.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ type ProjectConfig struct {
// Signature is used to configure the project's signature header versions
Signature *SignatureConfiguration `json:"signature"`

// RequestIDHeader is the outbound header name for the stable request id sent on webhook deliveries.
RequestIDHeader config.RequestIDHeaderProvider `json:"request_id_header,omitempty" valid:"optional"`

// MetaEvent is used to configure the project's meta events
MetaEvent *MetaEventConfiguration `json:"meta_event"`

Expand Down Expand Up @@ -96,6 +99,7 @@ func (pc *ProjectConfig) Transform() *datastore.ProjectConfig {
RateLimit: pc.RateLimit.Transform(),
Strategy: pc.Strategy.transform(),
Signature: pc.Signature.transform(),
RequestIDHeader: pc.RequestIDHeader,
MetaEvent: pc.MetaEvent.transform(),
CircuitBreaker: pc.CircuitBreaker,
}
Expand Down
1 change: 1 addition & 0 deletions api/testdb/seed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,7 @@ func SeedDefaultProjectWithSSL(db database.Database, orgID string, ssl *datastor
RetryCount: 2,
},
SSL: ssl,
RequestIDHeader: config.DefaultRequestIDHeader,
Signature: &datastore.SignatureConfiguration{
Header: config.DefaultSignatureHeader,
Versions: []datastore.SignatureVersion{
Expand Down
297 changes: 149 additions & 148 deletions api/ui/build/index.html
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,7 @@ const (
const (
RedisQueueProvider QueueProvider = "redis"
DefaultSignatureHeader SignatureHeaderProvider = "X-Convoy-Signature"
DefaultRequestIDHeader RequestIDHeaderProvider = "X-Convoy-Idempotency-Key"
PostgresDatabaseProvider DatabaseProvider = "postgres"
)

Expand All @@ -471,6 +472,7 @@ type (
AuthProvider string
QueueProvider string
SignatureHeaderProvider string
RequestIDHeaderProvider string
TracerProvider string
CacheProvider string
LimiterProvider string
Expand All @@ -483,6 +485,10 @@ func (s SignatureHeaderProvider) String() string {
return string(s)
}

func (r RequestIDHeaderProvider) String() string {
return string(r)
}

type ExecutionMode string

const (
Expand Down
35 changes: 35 additions & 0 deletions datastore/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ var (
ReplayAttacks: false,
DisableEndpoint: false,
AddEventIDTraceHeaders: false,
RequestIDHeader: config.DefaultRequestIDHeader,
SSL: &DefaultSSLConfig,
RateLimit: &DefaultRateLimitConfig,
Strategy: &DefaultStrategyConfig,
Expand Down Expand Up @@ -626,6 +627,7 @@ type ProjectConfig struct {
RateLimit *RateLimitConfiguration `json:"ratelimit" db:"ratelimit"`
Strategy *StrategyConfiguration `json:"strategy" db:"strategy"`
Signature *SignatureConfiguration `json:"signature" db:"signature"`
RequestIDHeader config.RequestIDHeaderProvider `json:"request_id_header"`
MetaEvent *MetaEventConfiguration `json:"meta_event" db:"meta_event"`
CircuitBreaker *CircuitBreakerConfiguration `json:"circuit_breaker" db:"circuit_breaker"`
}
Expand All @@ -651,6 +653,22 @@ func (p *ProjectConfig) GetSignatureConfig() SignatureConfiguration {
return SignatureConfiguration{}
}

func (p *ProjectConfig) GetRequestIDHeader() config.RequestIDHeaderProvider {
if p != nil && strings.TrimSpace(string(p.RequestIDHeader)) != "" {
return p.RequestIDHeader
}
return config.DefaultRequestIDHeader
}

func (p *ProjectConfig) UsesCustomRequestIDHeader() bool {
if p == nil {
return false
}

header := strings.TrimSpace(string(p.RequestIDHeader))
return header != "" && config.RequestIDHeaderProvider(header) != config.DefaultRequestIDHeader
}

func (p *ProjectConfig) GetSSLConfig() SSLConfiguration {
if p.SSL != nil {
return *p.SSL
Expand Down Expand Up @@ -759,6 +777,22 @@ func (o *Project) IsDeleted() bool { return o.DeletedAt.Valid }

func (o *Project) IsOwner(e *Endpoint) bool { return o.UID == e.ProjectID }

func (p *Project) ValidateOutgoingEventIdempotencyKey(idempotencyKey string) error {
if p == nil || p.Type != OutgoingProject {
return nil
}

if p.Config == nil || !p.Config.UsesCustomRequestIDHeader() {
return nil
}

if isStringEmpty(idempotencyKey) {
return ErrMissingIdempotencyKeyForCustomRequestIDHeader
}

return nil
}

var (
ErrSignupDisabled = errors.New("user registration is disabled")
ErrUserNotFound = errors.New("user not found")
Expand All @@ -778,6 +812,7 @@ var (
ErrNoActiveSecret = errors.New("no active secret found")
ErrSecretNotFound = errors.New("secret not found")
ErrMetaEventNotFound = errors.New("meta event not found")
ErrMissingIdempotencyKeyForCustomRequestIDHeader = errors.New("idempotency_key is required when a custom request_id_header is configured")
)

type AppMetadata struct {
Expand Down
48 changes: 48 additions & 0 deletions datastore/models_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"testing"
"time"

"github.com/frain-dev/convoy/config"
"github.com/stretchr/testify/require"
"gopkg.in/guregu/null.v4"
)
Expand Down Expand Up @@ -44,6 +45,53 @@ func TestProject_IsDeleted(t *testing.T) {
}
}

func TestProject_ValidateOutgoingEventIdempotencyKey(t *testing.T) {
customProject := &Project{
Type: OutgoingProject,
Config: &ProjectConfig{
RequestIDHeader: config.RequestIDHeaderProvider("Split-Request-ID"),
},
}

tests := []struct {
name string
project *Project
idempotencyKey string
wantErr error
}{
{
name: "custom_header_requires_idempotency_key",
project: customProject,
idempotencyKey: "",
wantErr: ErrMissingIdempotencyKeyForCustomRequestIDHeader,
},
{
name: "custom_header_with_idempotency_key",
project: customProject,
idempotencyKey: "stable-request-id",
},
{
name: "default_header_allows_missing_idempotency_key",
project: &Project{Type: OutgoingProject, Config: &ProjectConfig{}},
},
{
name: "incoming_project_allows_missing_idempotency_key",
project: &Project{Type: IncomingProject, Config: &ProjectConfig{RequestIDHeader: "Split-Request-ID"}},
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
err := tc.project.ValidateOutgoingEventIdempotencyKey(tc.idempotencyKey)
if tc.wantErr != nil {
require.ErrorIs(t, err, tc.wantErr)
return
}
require.NoError(t, err)
})
}
}

func TestProject_IsOwner(t *testing.T) {
tt := []struct {
name string
Expand Down
54 changes: 31 additions & 23 deletions internal/projects/impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ func projectConfigToCreateParams(id string, config *datastore.ProjectConfig) rep
cb := config.GetCircuitBreakerConfig()
ssl := config.GetSSLConfig()

requestIDHeader := string(config.GetRequestIDHeader())

return repo.CreateProjectConfigurationParams{
ID: common.StringToPgTextNullable(id),
SearchPolicy: common.StringToPgTextNullable(config.SearchPolicy),
Expand All @@ -140,6 +142,7 @@ func projectConfigToCreateParams(id string, config *datastore.ProjectConfig) rep
StrategyRetryCount: pgtype.Int4{Int32: int32(sc.RetryCount), Valid: true},
SignatureHeader: common.StringToPgTextNullable(string(sgc.Header)),
SignatureVersions: signatureVersionsToJSON(sgc.Versions),
RequestIDHeader: common.StringToPgTextNullable(requestIDHeader),
DisableEndpoint: pgtype.Bool{Bool: config.DisableEndpoint, Valid: true},
MetaEventsEnabled: pgtype.Bool{Bool: me.IsEnabled, Valid: true},
MetaEventsType: common.StringToPgTextNullable(string(me.Type)),
Expand All @@ -163,6 +166,7 @@ func projectConfigToUpdateParams(id string, config *datastore.ProjectConfig) rep
rlc := config.GetRateLimitConfig()
sc := config.GetStrategyConfig()
sgc := config.GetSignatureConfig()
requestIDHeader := string(config.GetRequestIDHeader())
me := config.GetMetaEventConfig()
cb := config.GetCircuitBreakerConfig()
ssl := config.GetSSLConfig()
Expand All @@ -178,6 +182,7 @@ func projectConfigToUpdateParams(id string, config *datastore.ProjectConfig) rep
StrategyRetryCount: pgtype.Int4{Int32: int32(sc.RetryCount), Valid: true},
SignatureHeader: common.StringToPgTextNullable(string(sgc.Header)),
SignatureVersions: signatureVersionsToJSON(sgc.Versions),
RequestIDHeader: common.StringToPgTextNullable(requestIDHeader),
DisableEndpoint: pgtype.Bool{Bool: config.DisableEndpoint, Valid: true},
MetaEventsEnabled: pgtype.Bool{Bool: me.IsEnabled, Valid: true},
MetaEventsType: common.StringToPgTextNullable(string(me.Type)),
Expand Down Expand Up @@ -205,29 +210,29 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
retainedEvents pgtype.Int4
createdAt, updatedAt, deletedAt pgtype.Timestamptz
// Config fields
searchPolicy pgtype.Text
strategyType, signatureHeader string
signatureVersions []byte
maxPayloadReadSize int32
multipleEndpointSubscriptions bool
replayAttacks bool
ratelimitCount int32
ratelimitDuration int32
strategyDuration int32
strategyRetryCount int32
disableEndpoint bool
sslEnforceSecureEndpoints pgtype.Bool
metaEventsEnabled bool
metaEventsType, metaEventsEventType pgtype.Text
metaEventsUrl, metaEventsSecret pgtype.Text
metaEventsPubSub []byte
cbSampleRate int32
cbErrorTimeout int32
cbFailureThreshold int32
cbSuccessThreshold int32
cbObservabilityWindow int32
cbMinimumRequestCount int32
cbConsecutiveFailureThreshold int32
searchPolicy pgtype.Text
strategyType, signatureHeader, requestIDHeader string
signatureVersions []byte
maxPayloadReadSize int32
multipleEndpointSubscriptions bool
replayAttacks bool
ratelimitCount int32
ratelimitDuration int32
strategyDuration int32
strategyRetryCount int32
disableEndpoint bool
sslEnforceSecureEndpoints pgtype.Bool
metaEventsEnabled bool
metaEventsType, metaEventsEventType pgtype.Text
metaEventsUrl, metaEventsSecret pgtype.Text
metaEventsPubSub []byte
cbSampleRate int32
cbErrorTimeout int32
cbFailureThreshold int32
cbSuccessThreshold int32
cbObservabilityWindow int32
cbMinimumRequestCount int32
cbConsecutiveFailureThreshold int32
)

switch r := row.(type) {
Expand All @@ -248,6 +253,7 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
strategyRetryCount = r.ConfigStrategyRetryCount
signatureHeader = r.ConfigSignatureHeader
signatureVersions = r.ConfigSignatureVersions
requestIDHeader = r.ConfigRequestIDHeader
disableEndpoint = r.ConfigDisableEndpoint
sslEnforceSecureEndpoints = r.ConfigSslEnforceSecureEndpoints
metaEventsEnabled = r.ConfigMetaEventsEnabled
Expand Down Expand Up @@ -280,6 +286,7 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
strategyRetryCount = r.ConfigStrategyRetryCount
signatureHeader = r.ConfigSignatureHeader
signatureVersions = r.ConfigSignatureVersions
requestIDHeader = r.ConfigRequestIDHeader
disableEndpoint = r.ConfigDisableEndpoint
sslEnforceSecureEndpoints = r.ConfigSslEnforceSecureEndpoints
metaEventsEnabled = r.ConfigMetaEventsEnabled
Expand Down Expand Up @@ -333,6 +340,7 @@ func rowToProject(row interface{}) (*datastore.Project, error) {
Header: config.SignatureHeaderProvider(signatureHeader),
Versions: jsonToSignatureVersions(signatureVersions),
},
RequestIDHeader: config.RequestIDHeaderProvider(requestIDHeader),
SSL: &datastore.SSLConfiguration{
EnforceSecureEndpoints: sslEnforceSecureEndpoints.Bool,
},
Expand Down
7 changes: 5 additions & 2 deletions internal/projects/queries.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ INSERT INTO convoy.project_configurations (
id, search_policy, max_payload_read_size,
replay_attacks_prevention_enabled, ratelimit_count,
ratelimit_duration, strategy_type, strategy_duration,
strategy_retry_count, signature_header, signature_versions,
strategy_retry_count, signature_header, signature_versions, request_id_header,
disable_endpoint, meta_events_enabled, meta_events_type,
meta_events_event_type, meta_events_url, meta_events_secret,
meta_events_pub_sub, ssl_enforce_secure_endpoints,
Expand All @@ -17,7 +17,7 @@ VALUES (
@id, @search_policy, @max_payload_read_size,
@replay_attacks_prevention_enabled, @ratelimit_count,
@ratelimit_duration, @strategy_type, @strategy_duration,
@strategy_retry_count, @signature_header, @signature_versions,
@strategy_retry_count, @signature_header, @signature_versions, @request_id_header,
@disable_endpoint, @meta_events_enabled, @meta_events_type,
@meta_events_event_type, @meta_events_url, @meta_events_secret,
@meta_events_pub_sub, @ssl_enforce_secure_endpoints,
Expand All @@ -37,6 +37,7 @@ UPDATE convoy.project_configurations SET
strategy_retry_count = @strategy_retry_count,
signature_header = @signature_header,
signature_versions = @signature_versions,
request_id_header = @request_id_header,
disable_endpoint = @disable_endpoint,
meta_events_enabled = @meta_events_enabled,
meta_events_type = @meta_events_type,
Expand Down Expand Up @@ -82,6 +83,7 @@ SELECT
c.strategy_retry_count AS "config_strategy_retry_count",
c.signature_header AS "config_signature_header",
c.signature_versions AS "config_signature_versions",
c.request_id_header AS "config_request_id_header",
c.disable_endpoint AS "config_disable_endpoint",
c.ssl_enforce_secure_endpoints AS "config_ssl_enforce_secure_endpoints",
c.meta_events_enabled AS "config_meta_events_enabled",
Expand Down Expand Up @@ -125,6 +127,7 @@ SELECT
c.strategy_retry_count AS "config_strategy_retry_count",
c.signature_header AS "config_signature_header",
c.signature_versions AS "config_signature_versions",
c.request_id_header AS "config_request_id_header",
c.disable_endpoint AS "config_disable_endpoint",
c.ssl_enforce_secure_endpoints AS "config_ssl_enforce_secure_endpoints",
c.meta_events_enabled AS "config_meta_events_enabled",
Expand Down
Loading
Loading