Skip to content

feat(stream): expose OTLP base URL on source API response (Phase 3) #2596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jordanrburger wants to merge 1 commit into from
Closed

feat(stream): expose OTLP base URL on source API response (Phase 3) #2596

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion api/stream/design.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1100,7 +1100,13 @@ var HTTPSource = Type("HTTPSource", func() {
Description("URL of the HTTP source. Contains secret used for authentication.")
Example("https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb")
})
Required("url")
Attribute("otlpUrl", String, func() {
Description("Base URL for the native OTLP/HTTP endpoint of this source. " +
"Use as OTEL_EXPORTER_OTLP_ENDPOINT; the OpenTelemetry SDK appends " +
"/v1/logs, /v1/metrics, or /v1/traces automatically.")
Example("https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb")
})
Required("url", "otlpUrl")
})

// Sink ----------------------------------------------------------------------------------------------------------------
Expand Down
3 changes: 2 additions & 1 deletion internal/pkg/service/stream/api/gen/http/stream/server/encode_decode.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions internal/pkg/service/stream/api/gen/http/stream/server/types.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions internal/pkg/service/stream/api/gen/stream/service.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 6 additions & 1 deletion internal/pkg/service/stream/api/mapper/aggregation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,13 @@ func (m *Mapper) NewAggregationSource(entity definition.Source) (*api.Aggregated
if err != nil {
return nil, err
}
otlpURL, err := entity.FormatOTLPSourceURL(m.httpSourcePublicURL.String())
if err != nil {
return nil, err
}
out.HTTP = &api.HTTPSource{
URL: u,
URL: u,
OtlpURL: otlpURL,
}
default:
return nil, svcerrors.NewBadRequestError(errors.Errorf(`unexpected "type" "%s"`, out.Type.String()))
Expand Down
8 changes: 7 additions & 1 deletion internal/pkg/service/stream/api/mapper/source_response.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,14 @@ func (m *Mapper) NewSourceResponse(entity definition.Source) (*api.Source, error
return nil, err
}

otlpURL, err := entity.FormatOTLPSourceURL(m.httpSourcePublicURL.String())
if err != nil {
return nil, err
}

out.HTTP = &api.HTTPSource{
URL: u,
URL: u,
OtlpURL: otlpURL,
}
default:
return nil, svcerrors.NewBadRequestError(errors.Errorf(`unexpected "type" "%s"`, out.Type.String()))
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/service/stream/api/openapi/openapi.json
View file
Open in desktop

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions internal/pkg/service/stream/api/openapi/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2099,6 +2099,7 @@ definitions:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -2302,6 +2303,7 @@ definitions:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -2485,6 +2487,7 @@ definitions:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -2950,15 +2953,21 @@ definitions:
title: HTTPSource
type: object
properties:
otlpUrl:
type: string
description: Base URL for the native OTLP/HTTP endpoint of this source. Use as OTEL_EXPORTER_OTLP_ENDPOINT; the OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.
example: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
url:
type: string
description: URL of the HTTP source. Contains secret used for authentication.
example: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
description: HTTP source details for "type" = "http".
example:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
required:
- url
- otlpUrl
Level:
title: Level
type: object
Expand Down Expand Up @@ -3774,6 +3783,7 @@ definitions:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -3846,6 +3856,7 @@ definitions:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -3899,6 +3910,7 @@ definitions:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/service/stream/api/openapi/openapi3.json
View file
Open in desktop

Large diffs are not rendered by default.

16 changes: 16 additions & 0 deletions internal/pkg/service/stream/api/openapi/openapi3.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ paths:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -340,6 +341,7 @@ paths:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -553,6 +555,7 @@ paths:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -2943,6 +2946,7 @@ paths:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -3456,6 +3460,7 @@ components:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -3645,6 +3650,7 @@ components:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -3872,6 +3878,7 @@ components:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -4555,15 +4562,21 @@ components:
HTTPSource:
type: object
properties:
otlpUrl:
type: string
description: Base URL for the native OTLP/HTTP endpoint of this source. Use as OTEL_EXPORTER_OTLP_ENDPOINT; the OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.
example: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
url:
type: string
description: URL of the HTTP source. Contains secret used for authentication.
example: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
description: HTTP source details for "type" = "http".
example:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
required:
- url
- otlpUrl
Level:
type: object
properties:
Expand Down Expand Up @@ -5632,6 +5645,7 @@ components:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -5747,6 +5761,7 @@ components:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down Expand Up @@ -5818,6 +5833,7 @@ components:
description: The source receives events from Github.
http:
url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
otlpUrl: https://stream-in.keboola.com/otlp/1234/my-source/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb
version:
number: 3
hash: f43e93acd97eceb3
Expand Down
17 changes: 17 additions & 0 deletions internal/pkg/service/stream/definition/source_base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,20 @@ func (s *Source) FormatHTTPSourceURL(httpSourcePublicURL string) (string, error)

return u.JoinPath("stream", s.ProjectID.String(), s.SourceID.String(), s.HTTP.Secret).String(), nil
}

// FormatOTLPSourceURL returns the OTLP/HTTP base URL for the source.
//
// OTel SDKs append the signal-specific suffix (/v1/logs, /v1/metrics,
// /v1/traces) automatically when given the base URL via
// OTEL_EXPORTER_OTLP_ENDPOINT, so this method returns the base only.
//
// The URL shares the same secret as the HTTP source — OTLP rides on top of
// the existing HTTP source, not a new source type.
func (s *Source) FormatOTLPSourceURL(httpSourcePublicURL string) (string, error) {
u, err := url.Parse(httpSourcePublicURL)
if err != nil {
return "", err
}

return u.JoinPath("otlp", s.ProjectID.String(), s.SourceID.String(), s.HTTP.Secret).String(), nil
}
53 changes: 53 additions & 0 deletions internal/pkg/service/stream/definition/source_base_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package definition_test

import (
"testing"

"github.com/keboola/keboola-sdk-go/v2/pkg/keboola"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition"
"github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key"
)

func TestSource_FormatOTLPSourceURL(t *testing.T) {
t.Parallel()

src := &definition.Source{
SourceKey: key.SourceKey{
BranchKey: key.BranchKey{ProjectID: keboola.ProjectID(12345), BranchID: keboola.BranchID(1)},
SourceID: "my-source",
},
HTTP: &definition.HTTPSource{Secret: "abc123secret"},
}

url, err := src.FormatOTLPSourceURL("https://stream.keboola.com")
require.NoError(t, err)
assert.Equal(t, "https://stream.keboola.com/otlp/12345/my-source/abc123secret", url)
}

func TestSource_FormatOTLPSourceURL_TrailingSlashInPublicURL(t *testing.T) {
t.Parallel()

src := &definition.Source{
SourceKey: key.SourceKey{
BranchKey: key.BranchKey{ProjectID: keboola.ProjectID(12345), BranchID: keboola.BranchID(1)},
SourceID: "my-source",
},
HTTP: &definition.HTTPSource{Secret: "s3cret"},
}

// JoinPath should normalize the slash regardless of the input form.
url, err := src.FormatOTLPSourceURL("https://stream.keboola.com/")
require.NoError(t, err)
assert.Equal(t, "https://stream.keboola.com/otlp/12345/my-source/s3cret", url)
}

func TestSource_FormatOTLPSourceURL_InvalidPublicURL(t *testing.T) {
t.Parallel()

src := &definition.Source{HTTP: &definition.HTTPSource{Secret: "x"}}
_, err := src.FormatOTLPSourceURL("\x7f://not a url")
require.Error(t, err)
}
3 changes: 2 additions & 1 deletion ...i/source/aggregation-002-ok/005-aggregation-limit-default-page-001/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
"name": "My Source 1",
"description": "",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source-1/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source-1/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-source-1/%s"
},
"version": {
"number": 1,
Expand Down
3 changes: 2 additions & 1 deletion test/stream/api/source/create-001-minimal-and-get/003-get-source/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"name": "My Source",
"description": "",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
},
"version": {
"number": 1,
Expand Down
3 changes: 2 additions & 1 deletion test/stream/api/source/create-002-with-id-and-get/003-get-source/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"name": "My Source",
"description": "",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source-custom-id/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source-custom-id/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-source-custom-id/%s"
},
"version": {
"number": 1,
Expand Down
3 changes: 2 additions & 1 deletion test/stream/api/source/create-003-weird-id-and-get/003-get-source/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"name": "My Source",
"description": "",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/weird-id/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/weird-id/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/weird-id/%s"
},
"version": {
"number": 1,
Expand Down
3 changes: 2 additions & 1 deletion ...ream/api/source/create-004-with-description-and-get/003-get-source/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"name": "My Source",
"description": "Some description ....",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
},
"version": {
"number": 1,
Expand Down
3 changes: 2 additions & 1 deletion test/stream/api/source/create-012-disable-enable/005-get-source/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"name": "My Source",
"description": "",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
},
"version": {
"number": 2,
Expand Down
3 changes: 2 additions & 1 deletion test/stream/api/source/create-012-disable-enable/008-get-source/expected-response.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
"name": "My Source",
"description": "",
"http": {
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
"url": "https://stream-in.keboola.local/stream/%%TEST_KBC_PROJECT_ID%%/my-source/%s",
"otlpUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-source/%s"
},
"version": {
"number": 3,
Expand Down
Loading
Loading