diff --git a/api/stream/design.go b/api/stream/design.go index 6782a8f000..2b7763d850 100644 --- a/api/stream/design.go +++ b/api/stream/design.go @@ -296,11 +296,42 @@ var _ = Service("stream", func() { Method("TestSource", func() { Meta("openapi:summary", "Test source payload mapping") - Description("Tests configured mapping of the source and its sinks.") + Description("Tests configured mapping of the source and its sinks against an example request body.\n\n" + + "For HTTP sources the body is the raw payload that an HTTP client would send (treated as a single record).\n\n" + + "For OTLP sources the body must be a single **flattened OTLP record** — a JSON object with the same shape that " + + "the source produces internally for each log record, metric data point, or span. Attributes, resource, and scope " + + "are nested objects (not dotted keys); reference them in column mappings as `Body('attributes')['user.id']`, " + + "`Body('resource')['service.name']`, `Body('scope')['name']`, etc. " + + "Example flattened log record:\n" + + "```json\n" + + "{\n" + + " \"timestamp\": \"2024-01-15T10:30:00Z\",\n" + + " \"observed_timestamp\": \"2024-01-15T10:30:00Z\",\n" + + " \"severity_number\": 9,\n" + + " \"severity_text\": \"INFO\",\n" + + " \"body\": \"User logged in\",\n" + + " \"flags\": 0,\n" + + " \"attributes\": {\"user.id\": \"user-123\"},\n" + + " \"resource\": {\"service.name\": \"auth-service\"},\n" + + " \"scope\": {\"name\": \"github.com/my/auth\", \"version\": \"1.2.3\"}\n" + + "}\n" + + "```\n" + + "Do not send a raw OTLP protobuf or the multi-record envelope produced by an OTel SDK — the test endpoint " + + "intentionally evaluates one already-flattened record so the response is deterministic. " + + "For OTLP sources, the `signal` query parameter selects which signal type the request simulates for sink " + + "routing (`logs` by default); sinks whose `allowedSignals` filter rejects that signal are skipped in the result.") Result(TestResult) Payload(TestSourceRequest) HTTP(func() { POST("/branches/{branchId}/sources/{sourceId}/test") + // Re-declare the enum here so Swagger 2.0 (openapi.yaml) emits it on + // the parameter schema — Goa propagates named-type enums to OpenAPI 3 + // automatically but drops them on the v2 parameter shape. + Param("signal", String, func() { + Description("OTLP signal type to simulate for sink routing.") + Enum("logs", "metrics", "traces") + Example("logs") + }) Meta("openapi:tag:test") Response(StatusOK) SourceNotFoundError() @@ -955,15 +986,86 @@ var SourceResponse = func() { var Source = Type("Source", func() { SourceResponse() + // HTTP and OTLP source examples are mutually exclusive — a real response + // carries exactly one of the type-specific blocks. Providing two named + // examples keeps the OpenAPI documentation from suggesting the impossible + // "type: http with both http and otlp blocks" shape that Goa would otherwise + // auto-synthesize from the optional attributes. + Example("http_source", func() { + Description("HTTP source response shape.") + Value(Val{ + "projectId": 1234, + "branchId": 5678, + "sourceId": "my-http-source", + "type": "http", + "name": "My HTTP Source", + "description": "", + "http": Val{ + "url": "https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX", + }, + "version": exampleVersion(), + "created": exampleCreated(), + }) + }) + Example("otlp_source", func() { + Description("OTLP source response shape.") + Value(Val{ + "projectId": 1234, + "branchId": 5678, + "sourceId": "my-otlp-source", + "type": "otlp", + "name": "My OTLP Source", + "description": "", + "otlp": Val{ + "url": "https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX", + "baseUrl": "https://stream-in.keboola.com/otlp/1234/my-otlp-source", + "secret": "EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX", + }, + "version": exampleVersion(), + "created": exampleCreated(), + }) + }) }) +// exampleVersion and exampleCreated build the required version/created blocks +// shared by every Source/AggregatedSource example so the OpenAPI documentation +// matches the schema (which marks both fields as required). +func exampleVersion() Val { + return Val{ + "number": 1, + "hash": "f43e93acd97eceb3", + "description": "New source.", + "at": "2024-01-15T10:00:00.000Z", + "by": Val{ + "type": "user", + "tokenId": "896455", + "tokenDesc": "john.green@company.com", + "userId": "578621", + "userName": "John Green", + }, + } +} + +func exampleCreated() Val { + return Val{ + "at": "2024-01-15T10:00:00.000Z", + "by": Val{ + "type": "user", + "tokenId": "896455", + "tokenDesc": "john.green@company.com", + "userId": "578621", + "userName": "John Green", + }, + } +} + var Sources = Type("Sources", ArrayOf(Source), func() { Description(fmt.Sprintf("List of sources, max %d sources per a branch.", source.MaxSourcesPerBranch)) }) var SourceType = Type("SourceType", String, func() { Meta("struct:field:type", "= definition.SourceType", "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition") - Enum(definition.SourceTypeHTTP.String()) + Enum(definition.SourceTypeHTTP.String(), definition.SourceTypeOTLP.String()) Example(definition.SourceTypeHTTP.String()) }) @@ -986,8 +1088,23 @@ var UpdateSourceRequest = Type("UpdateSourceRequest", func() { SourceFields(OpUpdate) }) +// OTLPSignal is the named enum reused by allowedSignals on sinks and the +// signal query param on the TestSource endpoint. Naming the type means +// validation errors report a clean field path instead of the doubly-indexed +// `allowedSignals[0].allowedSignals[*]` Goa produces for an anonymous enum. +var OTLPSignal = Type("OTLPSignal", String, func() { + Description("OTLP signal type — one of logs, metrics, or traces.") + Enum("logs", "metrics", "traces") + Example("logs") +}) + var TestSourceRequest = Type("TestSourceRequest", func() { SourceKeyRequest() + Attribute("signal", OTLPSignal, func() { + Description(`OTLP signal type to simulate for sink routing. Only applies to OTLP sources — ignored for HTTP sources. ` + + `Defaults to "logs" if omitted. Sinks whose ` + "`allowedSignals`" + ` filter rejects this signal are skipped in the result.`) + Example("logs") + }) }) var TestResult = Type("TestResult", func() { @@ -1074,11 +1191,14 @@ var SourceFields = func(op OperationType) { Example("The source receives events from Github.") }) - // HTTP - sub-definition - read-only + // Type-specific details - read-only if op == OpRead { Attribute("http", HTTPSource, func() { Description(fmt.Sprintf(`HTTP source details for "type" = "%s".`, definition.SourceTypeHTTP)) }) + Attribute("otlp", OTLPSource, func() { + Description(fmt.Sprintf(`OTLP source details for "type" = "%s".`, definition.SourceTypeOTLP)) + }) } // Required fields @@ -1098,11 +1218,35 @@ var HTTPSource = Type("HTTPSource", func() { Description(fmt.Sprintf(`HTTP source details for "type" = "%s".`, definition.SourceTypeHTTP)) Attribute("url", String, func() { Description("URL of the HTTP source. Contains secret used for authentication.") - Example("https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb") + Example("https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX") }) Required("url") }) +// OTLP Source---------------------------------------------------------------------------------------------------------- + +var OTLPSource = Type("OTLPSource", func() { + Description(fmt.Sprintf(`OTLP/HTTP source details for "type" = "%s".`, definition.SourceTypeOTLP)) + Attribute("url", String, func() { + Description("Endpoint URL with the secret embedded as the last path segment. " + + "Convenient for SDKs that authenticate by URL only. " + + "The OpenTelemetry SDK automatically appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — " + + "do not append a signal path yourself. Most SDK exporters reject or silently strip the suffix.") + Example("https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX") + }) + Attribute("baseUrl", String, func() { + Description("Endpoint URL without the secret. Use this together with the `secret` field via the " + + "`Authorization: Bearer ` header so the secret stays out of access/CDN/APM logs. " + + "The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.") + Example("https://stream-in.keboola.com/otlp/123/my-source") + }) + Attribute("secret", String, func() { + Description("48-character secret authenticating writes to this source. Send it as `Authorization: Bearer ` to the `baseUrl`.") + Example("EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX") + }) + Required("url", "baseUrl", "secret") +}) + // Sink ---------------------------------------------------------------------------------------------------------------- var SinkResponse = func() { @@ -1312,6 +1456,13 @@ var SinkFields = func(op OperationType) { Example("The sink stores records to a table.") }) + Attribute("allowedSignals", ArrayOf(OTLPSignal), func() { + Description(`Restricts the sink to specific OTLP signal types. ` + + `Empty (default) accepts all signals. ` + + `Only relevant for OTLP sources; HTTP sources ignore this field.`) + Example([]string{"logs"}) + }) + // Table sub-definition switch op { case OpRead: @@ -1573,6 +1724,44 @@ var AggregatedSource = Type("AggregatedSource", func() { SourceResponse() Attribute("sinks", AggregatedSinks) Required("sinks") + // Mutually exclusive type-specific blocks: see the Source definition above + // for the rationale behind these explicit named examples. + Example("http_source", func() { + Description("HTTP source with aggregated sink statistics.") + Value(Val{ + "projectId": 1234, + "branchId": 5678, + "sourceId": "my-http-source", + "type": "http", + "name": "My HTTP Source", + "description": "", + "http": Val{ + "url": "https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX", + }, + "version": exampleVersion(), + "created": exampleCreated(), + "sinks": []any{}, + }) + }) + Example("otlp_source", func() { + Description("OTLP source with aggregated sink statistics.") + Value(Val{ + "projectId": 1234, + "branchId": 5678, + "sourceId": "my-otlp-source", + "type": "otlp", + "name": "My OTLP Source", + "description": "", + "otlp": Val{ + "url": "https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX", + "baseUrl": "https://stream-in.keboola.com/otlp/1234/my-otlp-source", + "secret": "EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX", + }, + "version": exampleVersion(), + "created": exampleCreated(), + "sinks": []any{}, + }) + }) }) var AggregatedSinks = Type("AggregatedSinks", ArrayOf(AggregatedSink)) diff --git a/internal/pkg/service/stream/api/gen/http/stream/server/encode_decode.go b/internal/pkg/service/stream/api/gen/http/stream/server/encode_decode.go index e4a24e3bb6..1a494adeda 100644 --- a/internal/pkg/service/stream/api/gen/http/stream/server/encode_decode.go +++ b/internal/pkg/service/stream/api/gen/http/stream/server/encode_decode.go @@ -767,6 +767,7 @@ func DecodeTestSourceRequest(mux goahttp.Muxer, decoder func(*http.Request) goah var ( branchID string sourceID string + signal *string storageAPIToken string err error @@ -780,6 +781,15 @@ func DecodeTestSourceRequest(mux goahttp.Muxer, decoder func(*http.Request) goah if utf8.RuneCountInString(sourceID) > 48 { err = goa.MergeErrors(err, goa.InvalidLengthError("sourceId", sourceID, utf8.RuneCountInString(sourceID), 48, false)) } + signalRaw := r.URL.Query().Get("signal") + if signalRaw != "" { + signal = &signalRaw + } + if signal != nil { + if !(*signal == "logs" || *signal == "metrics" || *signal == "traces") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("signal", *signal, []any{"logs", "metrics", "traces"})) + } + } storageAPIToken = r.Header.Get("X-StorageApi-Token") if storageAPIToken == "" { err = goa.MergeErrors(err, goa.MissingFieldError("X-StorageApi-Token", "header")) @@ -787,7 +797,7 @@ func DecodeTestSourceRequest(mux goahttp.Muxer, decoder func(*http.Request) goah if err != nil { return payload, err } - payload = NewTestSourcePayload(branchID, sourceID, storageAPIToken) + payload = NewTestSourcePayload(branchID, sourceID, signal, storageAPIToken) if strings.Contains(payload.StorageAPIToken, " ") { // Remove authorization scheme prefix (e.g. "Bearer") cred := strings.SplitN(payload.StorageAPIToken, " ", 2)[1] @@ -3600,6 +3610,9 @@ func marshalStreamSourceToSourceResponseBody(v *stream.Source) *SourceResponseBo if v.HTTP != nil { res.HTTP = marshalStreamHTTPSourceToHTTPSourceResponseBody(v.HTTP) } + if v.Otlp != nil { + res.Otlp = marshalStreamOTLPSourceToOTLPSourceResponseBody(v.Otlp) + } if v.Version != nil { res.Version = marshalStreamVersionToVersionResponseBody(v.Version) } @@ -3629,6 +3642,21 @@ func marshalStreamHTTPSourceToHTTPSourceResponseBody(v *stream.HTTPSource) *HTTP return res } +// marshalStreamOTLPSourceToOTLPSourceResponseBody builds a value of type +// *OTLPSourceResponseBody from a value of type *stream.OTLPSource. +func marshalStreamOTLPSourceToOTLPSourceResponseBody(v *stream.OTLPSource) *OTLPSourceResponseBody { + if v == nil { + return nil + } + res := &OTLPSourceResponseBody{ + URL: v.URL, + BaseURL: v.BaseURL, + Secret: v.Secret, + } + + return res +} + // marshalStreamVersionToVersionResponseBody builds a value of type // *VersionResponseBody from a value of type *stream.Version. func marshalStreamVersionToVersionResponseBody(v *stream.Version) *VersionResponseBody { @@ -3940,6 +3968,12 @@ func marshalStreamSinkToSinkResponseBody(v *stream.Sink) *SinkResponseBody { Name: v.Name, Description: v.Description, } + if v.AllowedSignals != nil { + res.AllowedSignals = make([]string, len(v.AllowedSignals)) + for i, val := range v.AllowedSignals { + res.AllowedSignals[i] = string(val) + } + } if v.Table != nil { res.Table = marshalStreamTableSinkToTableSinkResponseBody(v.Table) } @@ -4066,6 +4100,9 @@ func marshalStreamAggregatedSourceToAggregatedSourceResponseBody(v *stream.Aggre if v.HTTP != nil { res.HTTP = marshalStreamHTTPSourceToHTTPSourceResponseBody(v.HTTP) } + if v.Otlp != nil { + res.Otlp = marshalStreamOTLPSourceToOTLPSourceResponseBody(v.Otlp) + } if v.Version != nil { res.Version = marshalStreamVersionToVersionResponseBody(v.Version) } @@ -4106,6 +4143,12 @@ func marshalStreamAggregatedSinkToAggregatedSinkResponseBody(v *stream.Aggregate Name: v.Name, Description: v.Description, } + if v.AllowedSignals != nil { + res.AllowedSignals = make([]string, len(v.AllowedSignals)) + for i, val := range v.AllowedSignals { + res.AllowedSignals[i] = string(val) + } + } if v.Table != nil { res.Table = marshalStreamTableSinkToTableSinkResponseBody(v.Table) } diff --git a/internal/pkg/service/stream/api/gen/http/stream/server/types.go b/internal/pkg/service/stream/api/gen/http/stream/server/types.go index 55a0c0d15e..975676e2ed 100644 --- a/internal/pkg/service/stream/api/gen/http/stream/server/types.go +++ b/internal/pkg/service/stream/api/gen/http/stream/server/types.go @@ -62,8 +62,11 @@ type CreateSinkRequestBody struct { // Human readable name of the sink. Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` // Description of the source. - Description *string `form:"description,omitempty" json:"description,omitempty" xml:"description,omitempty"` - Table *TableSinkCreateRequestBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` + Description *string `form:"description,omitempty" json:"description,omitempty" xml:"description,omitempty"` + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []string `form:"allowedSignals,omitempty" json:"allowedSignals,omitempty" xml:"allowedSignals,omitempty"` + Table *TableSinkCreateRequestBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` } // UpdateSinkSettingsRequestBody is the type of the "stream" service @@ -83,8 +86,11 @@ type UpdateSinkRequestBody struct { // Human readable name of the sink. Name *string `form:"name,omitempty" json:"name,omitempty" xml:"name,omitempty"` // Description of the source. - Description *string `form:"description,omitempty" json:"description,omitempty" xml:"description,omitempty"` - Table *TableSinkUpdateRequestBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` + Description *string `form:"description,omitempty" json:"description,omitempty" xml:"description,omitempty"` + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []string `form:"allowedSignals,omitempty" json:"allowedSignals,omitempty" xml:"allowedSignals,omitempty"` + Table *TableSinkUpdateRequestBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` } // APIVersionIndexResponseBody is the type of the "stream" service @@ -172,7 +178,9 @@ type GetSourceResponseBody struct { // Description of the source. Description string `form:"description" json:"description" xml:"description"` // HTTP source details for "type" = "http". - HTTP *HTTPSourceResponseBody `form:"http,omitempty" json:"http,omitempty" xml:"http,omitempty"` + HTTP *HTTPSourceResponseBody `form:"http,omitempty" json:"http,omitempty" xml:"http,omitempty"` + // OTLP source details for "type" = "otlp". + Otlp *OTLPSourceResponseBody `form:"otlp,omitempty" json:"otlp,omitempty" xml:"otlp,omitempty"` Version *VersionResponseBody `form:"version" json:"version" xml:"version"` Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` @@ -389,12 +397,15 @@ type GetSinkResponseBody struct { // Human readable name of the sink. Name string `form:"name" json:"name" xml:"name"` // Description of the source. - Description string `form:"description" json:"description" xml:"description"` - Table *TableSinkResponseBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` - Version *VersionResponseBody `form:"version" json:"version" xml:"version"` - Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` - Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` - Disabled *DisabledEntityResponseBody `form:"disabled,omitempty" json:"disabled,omitempty" xml:"disabled,omitempty"` + Description string `form:"description" json:"description" xml:"description"` + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []string `form:"allowedSignals,omitempty" json:"allowedSignals,omitempty" xml:"allowedSignals,omitempty"` + Table *TableSinkResponseBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` + Version *VersionResponseBody `form:"version" json:"version" xml:"version"` + Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` + Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` + Disabled *DisabledEntityResponseBody `form:"disabled,omitempty" json:"disabled,omitempty" xml:"disabled,omitempty"` } // GetSinkSettingsResponseBody is the type of the "stream" service @@ -1361,7 +1372,9 @@ type SourceResponseBody struct { // Description of the source. Description string `form:"description" json:"description" xml:"description"` // HTTP source details for "type" = "http". - HTTP *HTTPSourceResponseBody `form:"http,omitempty" json:"http,omitempty" xml:"http,omitempty"` + HTTP *HTTPSourceResponseBody `form:"http,omitempty" json:"http,omitempty" xml:"http,omitempty"` + // OTLP source details for "type" = "otlp". + Otlp *OTLPSourceResponseBody `form:"otlp,omitempty" json:"otlp,omitempty" xml:"otlp,omitempty"` Version *VersionResponseBody `form:"version" json:"version" xml:"version"` Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` @@ -1374,6 +1387,24 @@ type HTTPSourceResponseBody struct { URL string `form:"url" json:"url" xml:"url"` } +// OTLPSourceResponseBody is used to define fields on response body types. +type OTLPSourceResponseBody struct { + // Endpoint URL with the secret embedded as the last path segment. Convenient + // for SDKs that authenticate by URL only. The OpenTelemetry SDK automatically + // appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — do + // not append a signal path yourself. Most SDK exporters reject or silently + // strip the suffix. + URL string `form:"url" json:"url" xml:"url"` + // Endpoint URL without the secret. Use this together with the `secret` field + // via the `Authorization: Bearer ` header so the secret stays out of + // access/CDN/APM logs. The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or + // /v1/traces automatically. + BaseURL string `form:"baseUrl" json:"baseUrl" xml:"baseUrl"` + // 48-character secret authenticating writes to this source. Send it as + // `Authorization: Bearer ` to the `baseUrl`. + Secret string `form:"secret" json:"secret" xml:"secret"` +} + // VersionResponseBody is used to define fields on response body types. type VersionResponseBody struct { // Version number counted from 1. @@ -1517,12 +1548,15 @@ type SinkResponseBody struct { // Human readable name of the sink. Name string `form:"name" json:"name" xml:"name"` // Description of the source. - Description string `form:"description" json:"description" xml:"description"` - Table *TableSinkResponseBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` - Version *VersionResponseBody `form:"version" json:"version" xml:"version"` - Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` - Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` - Disabled *DisabledEntityResponseBody `form:"disabled,omitempty" json:"disabled,omitempty" xml:"disabled,omitempty"` + Description string `form:"description" json:"description" xml:"description"` + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []string `form:"allowedSignals,omitempty" json:"allowedSignals,omitempty" xml:"allowedSignals,omitempty"` + Table *TableSinkResponseBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` + Version *VersionResponseBody `form:"version" json:"version" xml:"version"` + Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` + Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` + Disabled *DisabledEntityResponseBody `form:"disabled,omitempty" json:"disabled,omitempty" xml:"disabled,omitempty"` } // LevelResponseBody is used to define fields on response body types. @@ -1579,7 +1613,9 @@ type AggregatedSourceResponseBody struct { // Description of the source. Description string `form:"description" json:"description" xml:"description"` // HTTP source details for "type" = "http". - HTTP *HTTPSourceResponseBody `form:"http,omitempty" json:"http,omitempty" xml:"http,omitempty"` + HTTP *HTTPSourceResponseBody `form:"http,omitempty" json:"http,omitempty" xml:"http,omitempty"` + // OTLP source details for "type" = "otlp". + Otlp *OTLPSourceResponseBody `form:"otlp,omitempty" json:"otlp,omitempty" xml:"otlp,omitempty"` Version *VersionResponseBody `form:"version" json:"version" xml:"version"` Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` @@ -1597,13 +1633,16 @@ type AggregatedSinkResponseBody struct { // Human readable name of the sink. Name string `form:"name" json:"name" xml:"name"` // Description of the source. - Description string `form:"description" json:"description" xml:"description"` - Table *TableSinkResponseBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` - Version *VersionResponseBody `form:"version" json:"version" xml:"version"` - Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` - Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` - Disabled *DisabledEntityResponseBody `form:"disabled,omitempty" json:"disabled,omitempty" xml:"disabled,omitempty"` - Statistics *AggregatedStatisticsResponseBody `form:"statistics,omitempty" json:"statistics,omitempty" xml:"statistics,omitempty"` + Description string `form:"description" json:"description" xml:"description"` + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []string `form:"allowedSignals,omitempty" json:"allowedSignals,omitempty" xml:"allowedSignals,omitempty"` + Table *TableSinkResponseBody `form:"table,omitempty" json:"table,omitempty" xml:"table,omitempty"` + Version *VersionResponseBody `form:"version" json:"version" xml:"version"` + Created *CreatedEntityResponseBody `form:"created" json:"created" xml:"created"` + Deleted *DeletedEntityResponseBody `form:"deleted,omitempty" json:"deleted,omitempty" xml:"deleted,omitempty"` + Disabled *DisabledEntityResponseBody `form:"disabled,omitempty" json:"disabled,omitempty" xml:"disabled,omitempty"` + Statistics *AggregatedStatisticsResponseBody `form:"statistics,omitempty" json:"statistics,omitempty" xml:"statistics,omitempty"` } // AggregatedStatisticsResponseBody is used to define fields on response body @@ -1782,6 +1821,9 @@ func NewGetSourceResponseBody(res *stream.Source) *GetSourceResponseBody { if res.HTTP != nil { body.HTTP = marshalStreamHTTPSourceToHTTPSourceResponseBody(res.HTTP) } + if res.Otlp != nil { + body.Otlp = marshalStreamOTLPSourceToOTLPSourceResponseBody(res.Otlp) + } if res.Version != nil { body.Version = marshalStreamVersionToVersionResponseBody(res.Version) } @@ -2033,6 +2075,12 @@ func NewGetSinkResponseBody(res *stream.Sink) *GetSinkResponseBody { Name: res.Name, Description: res.Description, } + if res.AllowedSignals != nil { + body.AllowedSignals = make([]string, len(res.AllowedSignals)) + for i, val := range res.AllowedSignals { + body.AllowedSignals[i] = string(val) + } + } if res.Table != nil { body.Table = marshalStreamTableSinkToTableSinkResponseBody(res.Table) } @@ -3145,10 +3193,14 @@ func NewUpdateSourceSettingsPayload(body *UpdateSourceSettingsRequestBody, branc } // NewTestSourcePayload builds a stream service TestSource endpoint payload. -func NewTestSourcePayload(branchID string, sourceID string, storageAPIToken string) *stream.TestSourcePayload { +func NewTestSourcePayload(branchID string, sourceID string, signal *string, storageAPIToken string) *stream.TestSourcePayload { v := &stream.TestSourcePayload{} v.BranchID = stream.BranchIDOrDefault(branchID) v.SourceID = stream.SourceID(sourceID) + if signal != nil { + tmpsignal := stream.OTLPSignal(*signal) + v.Signal = &tmpsignal + } v.StorageAPIToken = storageAPIToken return v @@ -3245,6 +3297,12 @@ func NewCreateSinkPayload(body *CreateSinkRequestBody, branchID string, sourceID sinkID := stream.SinkID(*body.SinkID) v.SinkID = &sinkID } + if body.AllowedSignals != nil { + v.AllowedSignals = make([]stream.OTLPSignal, len(body.AllowedSignals)) + for i, val := range body.AllowedSignals { + v.AllowedSignals[i] = stream.OTLPSignal(val) + } + } if body.Table != nil { v.Table = unmarshalTableSinkCreateRequestBodyToStreamTableSinkCreate(body.Table) } @@ -3338,6 +3396,12 @@ func NewUpdateSinkPayload(body *UpdateSinkRequestBody, branchID string, sourceID type_ := stream.SinkType(*body.Type) v.Type = &type_ } + if body.AllowedSignals != nil { + v.AllowedSignals = make([]stream.OTLPSignal, len(body.AllowedSignals)) + for i, val := range body.AllowedSignals { + v.AllowedSignals[i] = stream.OTLPSignal(val) + } + } if body.Table != nil { v.Table = unmarshalTableSinkUpdateRequestBodyToStreamTableSinkUpdate(body.Table) } @@ -3511,8 +3575,8 @@ func ValidateCreateSourceRequestBody(body *CreateSourceRequestBody, errContext [ } } if body.Type != nil { - if !(*body.Type == "http") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError(strings.Join(append(errContext, "type"), "."), *body.Type, []any{"http"})) + if !(*body.Type == "http" || *body.Type == "otlp") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError(strings.Join(append(errContext, "type"), "."), *body.Type, []any{"http", "otlp"})) } } if body.Name != nil { @@ -3537,8 +3601,8 @@ func ValidateCreateSourceRequestBody(body *CreateSourceRequestBody, errContext [ // UpdateSourceRequestBody func ValidateUpdateSourceRequestBody(body *UpdateSourceRequestBody, errContext []string) (err error) { if body.Type != nil { - if !(*body.Type == "http") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError(strings.Join(append(errContext, "type"), "."), *body.Type, []any{"http"})) + if !(*body.Type == "http" || *body.Type == "otlp") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError(strings.Join(append(errContext, "type"), "."), *body.Type, []any{"http", "otlp"})) } } if body.Name != nil { @@ -3612,6 +3676,12 @@ func ValidateCreateSinkRequestBody(body *CreateSinkRequestBody, errContext []str err = goa.MergeErrors(err, goa.InvalidLengthError(strings.Join(append(errContext, "description"), "."), *body.Description, utf8.RuneCountInString(*body.Description), 4096, false)) } } + for i, e := range body.AllowedSignals { + errContext := append(errContext, fmt.Sprintf(`allowedSignals[%d]`, i)) + if !(e == "logs" || e == "metrics" || e == "traces") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError(strings.Join(append(errContext, "allowedSignals[*]"), "."), e, []any{"logs", "metrics", "traces"})) + } + } if body.Table != nil { if err2 := ValidateTableSinkCreateRequestBody(body.Table, append(errContext, "table")); err2 != nil { err = goa.MergeErrors(err, err2) @@ -3657,6 +3727,12 @@ func ValidateUpdateSinkRequestBody(body *UpdateSinkRequestBody, errContext []str err = goa.MergeErrors(err, goa.InvalidLengthError(strings.Join(append(errContext, "description"), "."), *body.Description, utf8.RuneCountInString(*body.Description), 4096, false)) } } + for i, e := range body.AllowedSignals { + errContext := append(errContext, fmt.Sprintf(`allowedSignals[%d]`, i)) + if !(e == "logs" || e == "metrics" || e == "traces") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError(strings.Join(append(errContext, "allowedSignals[*]"), "."), e, []any{"logs", "metrics", "traces"})) + } + } if body.Table != nil { if err2 := ValidateTableSinkUpdateRequestBody(body.Table, append(errContext, "table")); err2 != nil { err = goa.MergeErrors(err, err2) diff --git a/internal/pkg/service/stream/api/gen/stream/service.go b/internal/pkg/service/stream/api/gen/stream/service.go index d02ac913dd..66bc6484a5 100644 --- a/internal/pkg/service/stream/api/gen/stream/service.go +++ b/internal/pkg/service/stream/api/gen/stream/service.go @@ -46,7 +46,37 @@ type Service interface { GetSourceSettings(context.Context, dependencies.SourceRequestScope, *GetSourceSettingsPayload) (res *SettingsResult, err error) // Update source settings. UpdateSourceSettings(context.Context, dependencies.SourceRequestScope, *UpdateSourceSettingsPayload) (res *Task, err error) - // Tests configured mapping of the source and its sinks. + // Tests configured mapping of the source and its sinks against an example + // request body. + + // For HTTP sources the body is the raw payload that an HTTP client would send + // (treated as a single record). + + // For OTLP sources the body must be a single **flattened OTLP record** — a + // JSON object with the same shape that the source produces internally for each + // log record, metric data point, or span. Attributes, resource, and scope are + // nested objects (not dotted keys); reference them in column mappings as + // `Body('attributes')['user.id']`, `Body('resource')['service.name']`, + // `Body('scope')['name']`, etc. Example flattened log record: + // ```json + // { + // "timestamp": "2024-01-15T10:30:00Z", + // "observed_timestamp": "2024-01-15T10:30:00Z", + // "severity_number": 9, + // "severity_text": "INFO", + // "body": "User logged in", + // "flags": 0, + // "attributes": {"user.id": "user-123"}, + // "resource": {"service.name": "auth-service"}, + // "scope": {"name": "github.com/my/auth", "version": "1.2.3"} + // } + // ``` + // Do not send a raw OTLP protobuf or the multi-record envelope produced by an + // OTel SDK — the test endpoint intentionally evaluates one already-flattened + // record so the response is deterministic. For OTLP sources, the `signal` + // query parameter selects which signal type the request simulates for sink + // routing (`logs` by default); sinks whose `allowedSignals` filter rejects + // that signal are skipped in the result. TestSource(context.Context, dependencies.SourceRequestScope, *TestSourcePayload, io.ReadCloser) (res *TestResult, err error) // Clears all statistics of the source. SourceStatisticsClear(context.Context, dependencies.SourceRequestScope, *SourceStatisticsClearPayload) (err error) @@ -135,12 +165,15 @@ type AggregatedSink struct { Name string // Description of the source. Description string - Table *TableSink - Version *Version - Created *CreatedEntity - Deleted *DeletedEntity - Disabled *DisabledEntity - Statistics *AggregatedStatistics + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []OTLPSignal + Table *TableSink + Version *Version + Created *CreatedEntity + Deleted *DeletedEntity + Disabled *DisabledEntity + Statistics *AggregatedStatistics } type AggregatedSinks []*AggregatedSink @@ -157,7 +190,9 @@ type AggregatedSource struct { // Description of the source. Description string // HTTP source details for "type" = "http". - HTTP *HTTPSource + HTTP *HTTPSource + // OTLP source details for "type" = "otlp". + Otlp *OTLPSource Version *Version Created *CreatedEntity Deleted *DeletedEntity @@ -227,7 +262,10 @@ type CreateSinkPayload struct { Name string // Description of the source. Description *string - Table *TableSinkCreate + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []OTLPSignal + Table *TableSinkCreate } // CreateSourcePayload is the payload type of the stream service CreateSource @@ -473,6 +511,27 @@ type ListSourcesPayload struct { Limit int } +// OTLP signal type — one of logs, metrics, or traces. +type OTLPSignal string + +// OTLP/HTTP source details for "type" = "otlp". +type OTLPSource struct { + // Endpoint URL with the secret embedded as the last path segment. Convenient + // for SDKs that authenticate by URL only. The OpenTelemetry SDK automatically + // appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — do + // not append a signal path yourself. Most SDK exporters reject or silently + // strip the suffix. + URL string + // Endpoint URL without the secret. Use this together with the `secret` field + // via the `Authorization: Bearer ` header so the secret stays out of + // access/CDN/APM logs. The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or + // /v1/traces automatically. + BaseURL string + // 48-character secret authenticating writes to this source. Send it as + // `Authorization: Bearer ` to the `baseUrl`. + Secret string +} + type PaginatedResponse struct { // Current limit. Limit int @@ -564,11 +623,14 @@ type Sink struct { Name string // Description of the source. Description string - Table *TableSink - Version *Version - Created *CreatedEntity - Deleted *DeletedEntity - Disabled *DisabledEntity + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []OTLPSignal + Table *TableSink + Version *Version + Created *CreatedEntity + Deleted *DeletedEntity + Disabled *DisabledEntity } type SinkFile struct { @@ -676,7 +738,9 @@ type Source struct { // Description of the source. Description string // HTTP source details for "type" = "http". - HTTP *HTTPSource + HTTP *HTTPSource + // OTLP source details for "type" = "otlp". + Otlp *OTLPSource Version *Version Created *CreatedEntity Deleted *DeletedEntity @@ -850,6 +914,10 @@ type TestSourcePayload struct { StorageAPIToken string BranchID BranchIDOrDefault SourceID SourceID + // OTLP signal type to simulate for sink routing. Only applies to OTLP sources + // — ignored for HTTP sources. Defaults to "logs" if omitted. Sinks whose + // `allowedSignals` filter rejects this signal are skipped in the result. + Signal *OTLPSignal } // UndeleteSinkPayload is the payload type of the stream service UndeleteSink @@ -883,7 +951,10 @@ type UpdateSinkPayload struct { Name *string // Description of the source. Description *string - Table *TableSinkUpdate + // Restricts the sink to specific OTLP signal types. Empty (default) accepts + // all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + AllowedSignals []OTLPSignal + Table *TableSinkUpdate } // UpdateSinkSettingsPayload is the payload type of the stream service diff --git a/internal/pkg/service/stream/api/mapper/aggregation.go b/internal/pkg/service/stream/api/mapper/aggregation.go index e8a543ac69..90d72eff97 100644 --- a/internal/pkg/service/stream/api/mapper/aggregation.go +++ b/internal/pkg/service/stream/api/mapper/aggregation.go @@ -55,9 +55,18 @@ func (m *Mapper) NewAggregationSource(entity definition.Source) (*api.Aggregated if err != nil { return nil, err } - out.HTTP = &api.HTTPSource{ - URL: u, + out.HTTP = &api.HTTPSource{URL: u} + case definition.SourceTypeOTLP: + publicURL := m.httpSourcePublicURL.String() + u, err := entity.FormatOTLPSourceURL(publicURL) + if err != nil { + return nil, err } + baseURL, err := entity.FormatOTLPSourceBaseURL(publicURL) + if err != nil { + return nil, err + } + out.Otlp = &api.OTLPSource{URL: u, BaseURL: baseURL, Secret: entity.OTLP.Secret} default: return nil, svcerrors.NewBadRequestError(errors.Errorf(`unexpected "type" "%s"`, out.Type.String())) } @@ -67,16 +76,17 @@ func (m *Mapper) NewAggregationSource(entity definition.Source) (*api.Aggregated func (m *Mapper) NewAggregationSinkResponse(entity repository.SinkWithStatistics) (*api.AggregatedSink, error) { out := &api.AggregatedSink{ - ProjectID: entity.ProjectID, - BranchID: entity.BranchID, - SourceID: entity.SourceID, - SinkID: entity.SinkID, - Name: entity.Name, - Description: entity.Description, - Created: m.NewCreatedResponse(entity.Created), - Version: m.NewVersionResponse(entity.Version), - Deleted: m.NewDeletedResponse(entity.SoftDeletable), - Disabled: m.NewDisabledResponse(entity.Switchable), + ProjectID: entity.ProjectID, + BranchID: entity.BranchID, + SourceID: entity.SourceID, + SinkID: entity.SinkID, + Name: entity.Name, + Description: entity.Description, + AllowedSignals: stringsToSignals(entity.AllowedSignals), + Created: m.NewCreatedResponse(entity.Created), + Version: m.NewVersionResponse(entity.Version), + Deleted: m.NewDeletedResponse(entity.SoftDeletable), + Disabled: m.NewDisabledResponse(entity.Switchable), } if entity.Statistics.Total != nil { diff --git a/internal/pkg/service/stream/api/mapper/sink_allowed_signals_test.go b/internal/pkg/service/stream/api/mapper/sink_allowed_signals_test.go new file mode 100644 index 0000000000..1e7ef36b4a --- /dev/null +++ b/internal/pkg/service/stream/api/mapper/sink_allowed_signals_test.go @@ -0,0 +1,190 @@ +package mapper_test + +import ( + "net/url" + "testing" + + "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + api "github.com/keboola/keboola-as-code/internal/pkg/service/stream/api/gen/stream" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/api/mapper" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/mapping/table/column" +) + +type testDeps struct{} + +func (testDeps) APIPublicURL() *url.URL { return mustParseURL("http://api.example.com") } +func (testDeps) HTTPSourcePublicURL() *url.URL { return mustParseURL("http://source.example.com") } + +func mustParseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + return u +} + +func newTestMapper() *mapper.Mapper { + return mapper.New(testDeps{}, config.Config{}) +} + +func newMinimalTablePayload() *api.TableSinkCreate { + tableType := definition.TableTypeKeboola + return &api.TableSinkCreate{ + Type: tableType, + TableID: "in.c-bucket.my-table", + Mapping: &api.TableMapping{ + Columns: api.TableColumns{ + {Type: column.Datetime{}.ColumnType(), Name: "datetime"}, + }, + }, + } +} + +func newSourceKey() key.SourceKey { + return key.SourceKey{ + BranchKey: key.BranchKey{ProjectID: 123, BranchID: 456}, + SourceID: "my-source", + } +} + +func newTableSink() *definition.TableSink { + return &definition.TableSink{ + Type: definition.TableTypeKeboola, + Keboola: &definition.KeboolaTable{ + TableID: keboola.MustParseTableID("in.c-bucket.my-table"), + }, + } +} + +func TestNewSinkEntity_AllowedSignals(t *testing.T) { + t.Parallel() + m := newTestMapper() + sourceKey := newSourceKey() + + t.Run("set_logs_only", func(t *testing.T) { + t.Parallel() + payload := &api.CreateSinkPayload{ + Name: "logs-sink", + Type: definition.SinkTypeTable, + AllowedSignals: []api.OTLPSignal{"logs"}, + Table: newMinimalTablePayload(), + } + entity, err := m.NewSinkEntity(sourceKey, payload) + require.NoError(t, err) + assert.Equal(t, []string{"logs"}, entity.AllowedSignals) + }) + + t.Run("set_multiple_signals", func(t *testing.T) { + t.Parallel() + payload := &api.CreateSinkPayload{ + Name: "multi-sink", + Type: definition.SinkTypeTable, + AllowedSignals: []api.OTLPSignal{"logs", "metrics"}, + Table: newMinimalTablePayload(), + } + entity, err := m.NewSinkEntity(sourceKey, payload) + require.NoError(t, err) + assert.Equal(t, []string{"logs", "metrics"}, entity.AllowedSignals) + }) + + t.Run("empty_means_all_signals", func(t *testing.T) { + t.Parallel() + payload := &api.CreateSinkPayload{ + Name: "all-sink", + Type: definition.SinkTypeTable, + Table: newMinimalTablePayload(), + } + entity, err := m.NewSinkEntity(sourceKey, payload) + require.NoError(t, err) + assert.Empty(t, entity.AllowedSignals) + }) +} + +func TestUpdateSinkEntity_AllowedSignals(t *testing.T) { + t.Parallel() + m := newTestMapper() + sinkType := definition.SinkTypeTable + + base := definition.Sink{ + SinkKey: key.SinkKey{SourceKey: newSourceKey(), SinkID: "my-sink"}, + Type: definition.SinkTypeTable, + Name: "My Sink", + Table: newTableSink(), + } + + t.Run("set_signals", func(t *testing.T) { + t.Parallel() + entity := base + payload := &api.UpdateSinkPayload{ + Type: &sinkType, + AllowedSignals: []api.OTLPSignal{"traces"}, + } + updated, err := m.UpdateSinkEntity(entity, payload) + require.NoError(t, err) + assert.Equal(t, []string{"traces"}, updated.AllowedSignals) + }) + + t.Run("nil_means_no_change", func(t *testing.T) { + t.Parallel() + entity := base + entity.AllowedSignals = []string{"logs"} + payload := &api.UpdateSinkPayload{ + Type: &sinkType, + AllowedSignals: nil, + } + updated, err := m.UpdateSinkEntity(entity, payload) + require.NoError(t, err) + assert.Equal(t, []string{"logs"}, updated.AllowedSignals) + }) + + t.Run("empty_slice_clears_filter", func(t *testing.T) { + t.Parallel() + entity := base + entity.AllowedSignals = []string{"logs"} + payload := &api.UpdateSinkPayload{ + Type: &sinkType, + AllowedSignals: []api.OTLPSignal{}, + } + updated, err := m.UpdateSinkEntity(entity, payload) + require.NoError(t, err) + assert.Empty(t, updated.AllowedSignals) + }) +} + +func TestNewSinkResponse_AllowedSignals(t *testing.T) { + t.Parallel() + m := newTestMapper() + + t.Run("signals_present_in_response", func(t *testing.T) { + t.Parallel() + entity := definition.Sink{ + SinkKey: key.SinkKey{SourceKey: newSourceKey(), SinkID: "my-sink"}, + Type: definition.SinkTypeTable, + Name: "My Sink", + AllowedSignals: []string{"logs", "metrics"}, + Table: newTableSink(), + } + resp, err := m.NewSinkResponse(entity) + require.NoError(t, err) + assert.Equal(t, []api.OTLPSignal{"logs", "metrics"}, resp.AllowedSignals) + }) + + t.Run("empty_signals_in_response", func(t *testing.T) { + t.Parallel() + entity := definition.Sink{ + SinkKey: key.SinkKey{SourceKey: newSourceKey(), SinkID: "my-sink"}, + Type: definition.SinkTypeTable, + Name: "My Sink", + Table: newTableSink(), + } + resp, err := m.NewSinkResponse(entity) + require.NoError(t, err) + assert.Empty(t, resp.AllowedSignals) + }) +} diff --git a/internal/pkg/service/stream/api/mapper/sink_request.go b/internal/pkg/service/stream/api/mapper/sink_request.go index d6d236e216..1ea70dd976 100644 --- a/internal/pkg/service/stream/api/mapper/sink_request.go +++ b/internal/pkg/service/stream/api/mapper/sink_request.go @@ -34,6 +34,12 @@ func (m *Mapper) NewSinkEntity(parent key.SourceKey, payload *api.CreateSinkPayl entity.Description = *payload.Description } + // AllowedSignals is optional — empty means accept all signals. + // Convert from the Goa-typed enum slice to plain []string for storage. + if len(payload.AllowedSignals) > 0 { + entity.AllowedSignals = signalsToStrings(payload.AllowedSignals) + } + // Sink type entity.Type = payload.Type switch entity.Type { @@ -61,6 +67,11 @@ func (m *Mapper) UpdateSinkEntity(entity definition.Sink, payload *api.UpdateSin entity.Description = *payload.Description } + // AllowedSignals — nil means "don't change", empty slice means "clear the filter" + if payload.AllowedSignals != nil { + entity.AllowedSignals = signalsToStrings(payload.AllowedSignals) + } + // Type if payload.Type != nil { entity.Type = *payload.Type @@ -118,6 +129,29 @@ func (m *Mapper) newTableSinkEntity(payload *api.CreateSinkPayload) (entity defi return entity, err } +// signalsToStrings converts the Goa-typed []api.OTLPSignal payload field into +// the plain []string used by the persisted entity. The reverse conversion is +// stringsToSignals immediately below. +func signalsToStrings(signals []api.OTLPSignal) []string { + out := make([]string, len(signals)) + for i, s := range signals { + out[i] = string(s) + } + return out +} + +// stringsToSignals is the reverse conversion used when building API responses. +func stringsToSignals(signals []string) []api.OTLPSignal { + if signals == nil { + return nil + } + out := make([]api.OTLPSignal, len(signals)) + for i, s := range signals { + out[i] = api.OTLPSignal(s) + } + return out +} + func (m *Mapper) newTableSinkMappingEntity(payload *api.TableMapping) (entity table.Mapping, err error) { // User has to specify table mapping definition if payload == nil { diff --git a/internal/pkg/service/stream/api/mapper/sink_response.go b/internal/pkg/service/stream/api/mapper/sink_response.go index 35b822ce0f..bb292add8f 100644 --- a/internal/pkg/service/stream/api/mapper/sink_response.go +++ b/internal/pkg/service/stream/api/mapper/sink_response.go @@ -17,16 +17,17 @@ import ( func (m *Mapper) NewSinkResponse(entity definition.Sink) (*api.Sink, error) { out := &api.Sink{ - ProjectID: entity.ProjectID, - BranchID: entity.BranchID, - SourceID: entity.SourceID, - SinkID: entity.SinkID, - Name: entity.Name, - Description: entity.Description, - Created: m.NewCreatedResponse(entity.Created), - Version: m.NewVersionResponse(entity.Version), - Deleted: m.NewDeletedResponse(entity.SoftDeletable), - Disabled: m.NewDisabledResponse(entity.Switchable), + ProjectID: entity.ProjectID, + BranchID: entity.BranchID, + SourceID: entity.SourceID, + SinkID: entity.SinkID, + Name: entity.Name, + Description: entity.Description, + AllowedSignals: stringsToSignals(entity.AllowedSignals), + Created: m.NewCreatedResponse(entity.Created), + Version: m.NewVersionResponse(entity.Version), + Deleted: m.NewDeletedResponse(entity.SoftDeletable), + Disabled: m.NewDisabledResponse(entity.Switchable), } // Type diff --git a/internal/pkg/service/stream/api/mapper/source_request.go b/internal/pkg/service/stream/api/mapper/source_request.go index 4051b3e3b0..548e34f75d 100644 --- a/internal/pkg/service/stream/api/mapper/source_request.go +++ b/internal/pkg/service/stream/api/mapper/source_request.go @@ -39,6 +39,10 @@ func (m *Mapper) NewSourceEntity(parent key.BranchKey, payload *api.CreateSource entity.HTTP = &definition.HTTPSource{ Secret: idgenerator.StreamHTTPSourceSecret(), } + case definition.SourceTypeOTLP: + entity.OTLP = &definition.OTLPSource{ + Secret: idgenerator.StreamHTTPSourceSecret(), + } default: return definition.Source{}, svcerrors.NewBadRequestError(errors.Errorf(`unexpected "type" "%s"`, payload.Type.String())) } @@ -62,7 +66,9 @@ func (m *Mapper) UpdateSourceEntity(entity definition.Source, payload *api.Updat entity.Type = *payload.Type } - // Type specific updates + // Type-specific updates. Only the block matching the active type is kept + // so that switching type drops the previous type's stale secret/config and + // the persisted entity always carries exactly one type-specific block. switch entity.Type { case definition.SourceTypeHTTP: if entity.HTTP == nil { @@ -71,6 +77,15 @@ func (m *Mapper) UpdateSourceEntity(entity definition.Source, payload *api.Updat if entity.HTTP.Secret == "" { entity.HTTP.Secret = idgenerator.StreamHTTPSourceSecret() } + entity.OTLP = nil + case definition.SourceTypeOTLP: + if entity.OTLP == nil { + entity.OTLP = &definition.OTLPSource{} + } + if entity.OTLP.Secret == "" { + entity.OTLP.Secret = idgenerator.StreamHTTPSourceSecret() + } + entity.HTTP = nil default: return definition.Source{}, svcerrors.NewBadRequestError(errors.Errorf(`unexpected "type" "%s"`, payload.Type.String())) } diff --git a/internal/pkg/service/stream/api/mapper/source_response.go b/internal/pkg/service/stream/api/mapper/source_response.go index 460607ad63..3dba252fdc 100644 --- a/internal/pkg/service/stream/api/mapper/source_response.go +++ b/internal/pkg/service/stream/api/mapper/source_response.go @@ -38,10 +38,18 @@ func (m *Mapper) NewSourceResponse(entity definition.Source) (*api.Source, error if err != nil { return nil, err } - - out.HTTP = &api.HTTPSource{ - URL: u, + out.HTTP = &api.HTTPSource{URL: u} + case definition.SourceTypeOTLP: + publicURL := m.httpSourcePublicURL.String() + u, err := entity.FormatOTLPSourceURL(publicURL) + if err != nil { + return nil, err + } + baseURL, err := entity.FormatOTLPSourceBaseURL(publicURL) + if err != nil { + return nil, err } + out.Otlp = &api.OTLPSource{URL: u, BaseURL: baseURL, Secret: entity.OTLP.Secret} default: return nil, svcerrors.NewBadRequestError(errors.Errorf(`unexpected "type" "%s"`, out.Type.String())) } diff --git a/internal/pkg/service/stream/api/openapi/openapi.json b/internal/pkg/service/stream/api/openapi/openapi.json index 66eba95172..841302a8fe 100644 --- a/internal/pkg/service/stream/api/openapi/openapi.json +++ b/internal/pkg/service/stream/api/openapi/openapi.json @@ -1 +1 @@ -{"swagger":"2.0","info":{"title":"Stream Service","description":"A service for continuously importing data to the Keboola platform.","license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0"},"version":"1.0"},"host":"stream.keboola.com","consumes":["application/json"],"produces":["application/json"],"paths":{"/":{"get":{"tags":["documentation"],"summary":"Redirect to /v1","description":"Redirect to /v1.","operationId":"ApiRootIndex","responses":{"301":{"description":"Moved Permanently response."}},"schemes":["https"]}},"/v1":{"get":{"tags":["documentation"],"summary":"List API name and link to documentation.","description":"List API name and link to documentation.","operationId":"ApiVersionIndex","responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/ServiceDetail","required":["api","documentation"]}}},"schemes":["https"]}},"/v1/branches/{branchId}/aggregation/sources":{"get":{"tags":["internal"],"summary":"Aggregation endpoint for sources","description":"Details about sources for the UI.","operationId":"AggregationSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/AggregatedSourcesResult","required":["projectId","branchId","page","sources"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources":{"get":{"tags":["configuration"],"summary":"List all sources","description":"List all sources in the branch.","operationId":"ListSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SourcesList","required":["projectId","branchId","page","sources"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"post":{"tags":["configuration"],"summary":"Create source","description":"Create a new source in the branch.","operationId":"CreateSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"CreateSourceRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamCreateSourceRequestBody","required":["type","name"]}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"409":{"description":"Conflict response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"422":{"description":"Unprocessable Entity response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/deleted":{"get":{"tags":["configuration"],"summary":"List all deleted sources","description":"List all deleted sources in the branch.","operationId":"ListDeletedSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SourcesList","required":["projectId","branchId","page","sources"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}":{"get":{"tags":["configuration"],"summary":"Get source","description":"Get the source definition.","operationId":"GetSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Source","required":["projectId","branchId","sourceId","type","name","description","version","created"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"delete":{"tags":["configuration"],"summary":"Delete source","description":"Delete the source.","operationId":"DeleteSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update source","description":"Update the source.","operationId":"UpdateSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"UpdateSourceRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSourceRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/disable":{"put":{"tags":["configuration"],"summary":"Disable source","description":"Disables the source.","operationId":"DisableSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/enable":{"put":{"tags":["configuration"],"summary":"Enable source","description":"Enables the source.","operationId":"EnableSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/settings":{"get":{"tags":["configuration"],"summary":"Get source settings","description":"Get source settings.","operationId":"GetSourceSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SettingsResult"}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update source settings","description":"Update source settings.","operationId":"UpdateSourceSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"UpdateSourceSettingsRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSourceSettingsRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks":{"get":{"tags":["configuration"],"summary":"List sinks","description":"List all sinks in the source.","operationId":"ListSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinksList","required":["projectId","branchId","sourceId","page","sinks"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"post":{"tags":["configuration"],"summary":"Create sink","description":"Create a new sink in the source.","operationId":"CreateSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"CreateSinkRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamCreateSinkRequestBody","required":["type","name"]}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"409":{"description":"Conflict response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"422":{"description":"Unprocessable Entity response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/deleted":{"get":{"tags":["configuration"],"summary":"List deleted sinks","description":"List all deleted sinks in the source.","operationId":"ListDeletedSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinksList","required":["projectId","branchId","sourceId","page","sinks"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}":{"get":{"tags":["configuration"],"summary":"Get sink","description":"Get the sink definition.","operationId":"GetSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Sink","required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"delete":{"tags":["configuration"],"summary":"Delete sink","description":"Delete the sink.","operationId":"DeleteSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update sink","description":"Update the sink.","operationId":"UpdateSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"UpdateSinkRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSinkRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/disable":{"put":{"tags":["configuration"],"summary":"Disable sink","description":"Disables the sink.","operationId":"DisableSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/enable":{"put":{"tags":["configuration"],"summary":"Enable sink","description":"Enables the sink.","operationId":"EnableSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/settings":{"get":{"tags":["configuration"],"summary":"Get sink settings","description":"Get the sink settings.","operationId":"GetSinkSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SettingsResult"}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update sink settings","description":"Update sink settings.","operationId":"UpdateSinkSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"UpdateSinkSettingsRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSinkSettingsRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear sink statistics","description":"Clears all statistics of the sink.","operationId":"SinkStatisticsClear","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response."},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/files":{"get":{"tags":["configuration"],"summary":"Sink files statistics","description":"Get files statistics of the sink.","operationId":"SinkStatisticsFiles","parameters":[{"name":"failedFiles","in":"query","description":"Filter for not imported files. If set to true, only not imported files will be included.","required":false,"type":"boolean","default":false},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinkStatisticsFilesResult","required":["files"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/total":{"get":{"tags":["configuration"],"summary":"Sink statistics total","description":"Get total statistics of the sink.","operationId":"SinkStatisticsTotal","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinkStatisticsTotalResult","required":["levels","total"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete sink","description":"Undelete the sink.","operationId":"UndeleteSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions":{"get":{"tags":["configuration"],"summary":"List sink versions","description":"List all sink versions.","operationId":"ListSinkVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/EntityVersions","required":["versions","page"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Sink version detail","description":"Sink version detail.","operationId":"SinkVersionDetail","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Version","required":["number","hash","at","by","description"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback sink version","description":"Rollback sink version.","operationId":"RollbackSinkVersion","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear source statistics","description":"Clears all statistics of the source.","operationId":"SourceStatisticsClear","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response."},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/test":{"post":{"tags":["test"],"summary":"Test source payload mapping","description":"Tests configured mapping of the source and its sinks.","operationId":"TestSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/TestResult","required":["projectId","branchId","sourceId","tables"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"422":{"description":"Unprocessable Entity response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete source","description":"Undelete the source.","operationId":"UndeleteSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions":{"get":{"tags":["configuration"],"summary":"List source versions","description":"List all source versions.","operationId":"ListSourceVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/EntityVersions","required":["versions","page"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Source version detail","description":"Source version detail.","operationId":"SourceVersionDetail","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Version","required":["number","hash","at","by","description"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback source version","description":"Rollback source version.","operationId":"RollbackSourceVersion","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/documentation/openapi.json":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 JSON Specification","operationId":"OpenapiJson","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/documentation/openapi.yaml":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 YAML Specification","operationId":"OpenapiYaml","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/documentation/openapi3.json":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 JSON Specification","operationId":"Openapi3Json","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/documentation/openapi3.yaml":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 YAML Specification","operationId":"Openapi3Yaml","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/tasks/{taskId}":{"get":{"tags":["configuration"],"summary":"Get task","description":"Get details of a task.","operationId":"GetTask","parameters":[{"name":"taskId","in":"path","description":"Unique ID of the task.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}}},"definitions":{"AggregatedSink":{"title":"AggregatedSink","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","example":123,"format":"int64"},"sinkId":{"type":"string","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"statistics":{"$ref":"#/definitions/AggregatedStatistics"},"table":{"$ref":"#/definitions/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/definitions/Version"}},"description":"A mapping from imported data to a destination table.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"AggregatedSource":{"title":"AggregatedSource","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"http":{"$ref":"#/definitions/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"projectId":{"type":"integer","example":123,"format":"int64"},"sinks":{"type":"array","items":{"$ref":"#/definitions/AggregatedSink"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]},"version":{"$ref":"#/definitions/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]},"required":["projectId","branchId","sourceId","type","name","description","version","created","sinks"]},"AggregatedSourcesResult":{"title":"AggregatedSourcesResult","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"page":{"$ref":"#/definitions/PaginatedResponse"},"projectId":{"type":"integer","example":123,"format":"int64"},"sources":{"type":"array","items":{"$ref":"#/definitions/AggregatedSource"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]}]}},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]}]},"required":["projectId","branchId","page","sources"]},"AggregatedStatistics":{"title":"AggregatedStatistics","type":"object","properties":{"files":{"type":"array","items":{"$ref":"#/definitions/SinkFile"},"example":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"levels":{"$ref":"#/definitions/Levels"},"total":{"$ref":"#/definitions/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["total","levels","files"]},"By":{"title":"By","type":"object","properties":{"tokenDesc":{"type":"string","description":"Description of the token.","example":"john.green@company.com"},"tokenId":{"type":"string","description":"ID of the token.","example":"896455"},"type":{"type":"string","description":"Date and time of deletion.","example":"user","enum":["system","user"]},"userId":{"type":"string","description":"ID of the user.","example":"578621"},"userName":{"type":"string","description":"Name of the user.","example":"John Green"}},"description":"Information about the operation actor.","example":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"required":["type"]},"CreatedEntity":{"title":"CreatedEntity","type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"}},"description":"Information about the entity creation.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DeletedEntity":{"title":"DeletedEntity","type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"}},"description":"Information about the deleted entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DisabledEntity":{"title":"DisabledEntity","type":"object","properties":{"at":{"type":"string","description":"Date and time of disabling.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"},"reason":{"type":"string","description":"Why was the entity disabled?","example":"Disabled for recurring problems."}},"description":"Information about the disabled entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"required":["at","by","reason"]},"EntityVersions":{"title":"EntityVersions","type":"object","properties":{"page":{"$ref":"#/definitions/PaginatedResponse"},"versions":{"type":"array","items":{"$ref":"#/definitions/Version"},"example":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}]}},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}},"required":["versions","page"]},"GenericError":{"title":"GenericError","type":"object","properties":{"error":{"type":"string","description":"Name of error.","example":"stream.internalError"},"message":{"type":"string","description":"Error message.","example":"Internal Error"},"statusCode":{"type":"integer","description":"HTTP status code.","example":500,"format":"int64"}},"description":"Source already exists in the branch.","example":{"statusCode":409,"error":"stream.api.sourceAlreadyExists","message":"Source already exists in the branch."},"required":["statusCode","error","message"]},"HTTPSource":{"title":"HTTPSource","type":"object","properties":{"url":{"type":"string","description":"URL of the HTTP source. Contains secret used for authentication.","example":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"}},"description":"HTTP source details for \"type\" = \"http\".","example":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"required":["url"]},"Level":{"title":"Level","type":"object","properties":{"compressedSize":{"type":"integer","description":"Compressed size of data in bytes.","example":1,"format":"int64"},"firstRecordAt":{"type":"string","description":"Timestamp of the first received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"lastRecordAt":{"type":"string","description":"Timestamp of the last received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"recordsCount":{"type":"integer","example":1,"format":"int64"},"uncompressedSize":{"type":"integer","description":"Uncompressed size of data in bytes.","example":1,"format":"int64"}},"example":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"required":["recordsCount","compressedSize","uncompressedSize"]},"Levels":{"title":"Levels","type":"object","properties":{"local":{"$ref":"#/definitions/Level"},"staging":{"$ref":"#/definitions/Level"},"target":{"$ref":"#/definitions/Level"}},"example":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"PaginatedResponse":{"title":"PaginatedResponse","type":"object","properties":{"afterId":{"type":"string","description":"Current offset.","example":"my-object-123"},"lastId":{"type":"string","description":"ID of the last record in the response.","example":"my-object-456"},"limit":{"type":"integer","description":"Current limit.","example":100,"format":"int64"},"totalCount":{"type":"integer","description":"Total count of all records.","example":1000,"format":"int64"}},"example":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"required":["afterId","limit","lastId","totalCount"]},"ServiceDetail":{"title":"ServiceDetail","type":"object","properties":{"api":{"type":"string","description":"Name of the API","example":"stream"},"documentation":{"type":"string","description":"URL of the API documentation.","example":"https://stream.keboola.com/v1/documentation"}},"example":{"api":"stream","documentation":"https://stream.keboola.com/v1/documentation"},"required":["api","documentation"]},"SettingPatch":{"title":"SettingPatch","type":"object","properties":{"key":{"type":"string","description":"Key path.","example":"some.service.limit","minLength":1},"value":{"description":"A new key value. Use null to reset the value to the default value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","value":"1m20s"},"required":["key"]},"SettingResult":{"title":"SettingResult","type":"object","properties":{"defaultValue":{"description":"Default value.","example":"30s"},"description":{"type":"string","description":"Key description.","example":"Minimal interval between uploads."},"key":{"type":"string","description":"Key path.","example":"some.service.limit"},"overwritten":{"type":"boolean","description":"True, if the default value is locally overwritten.","example":true},"protected":{"type":"boolean","description":"True, if only a super admin can modify the key.","example":false},"type":{"type":"string","description":"Value type.","example":"string","enum":["string","int","float","bool","[]string","[]int","[]float"]},"validation":{"type":"string","description":"Validation rules as a string definition.","example":"minDuration=15s"},"value":{"description":"Actual value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"},"required":["key","type","description","value","defaultValue","overwritten","protected"]},"SettingsResult":{"title":"SettingsResult","type":"object","properties":{"settings":{"type":"array","items":{"$ref":"#/definitions/SettingResult"},"example":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"Sink":{"title":"Sink","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","example":123,"format":"int64"},"sinkId":{"type":"string","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/definitions/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/definitions/Version"}},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"SinkFile":{"title":"SinkFile","type":"object","properties":{"closingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"openedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAfter":{"type":"string","description":"Next attempt time.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAttempt":{"type":"integer","description":"Number of failed attempts.","example":3,"format":"int64"},"retryReason":{"type":"string","description":"Reason of the last failed attempt.","example":"network problem"},"state":{"type":"string","example":"writing","enum":["writing","closing","importing","imported"]},"statistics":{"$ref":"#/definitions/SinkFileStatistics"}},"example":{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}},"required":["state","openedAt"]},"SinkFileStatistics":{"title":"SinkFileStatistics","type":"object","properties":{"levels":{"$ref":"#/definitions/Levels"},"total":{"$ref":"#/definitions/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["total","levels"]},"SinkStatisticsFilesResult":{"title":"SinkStatisticsFilesResult","type":"object","properties":{"files":{"type":"array","items":{"$ref":"#/definitions/SinkFile"},"example":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}},"example":{"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["files"]},"SinkStatisticsTotalResult":{"title":"SinkStatisticsTotalResult","type":"object","properties":{"levels":{"$ref":"#/definitions/Levels"},"total":{"$ref":"#/definitions/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["levels","total"]},"SinksList":{"title":"SinksList","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"page":{"$ref":"#/definitions/PaginatedResponse"},"projectId":{"type":"integer","example":123,"format":"int64"},"sinks":{"type":"array","items":{"$ref":"#/definitions/Sink"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"required":["projectId","branchId","sourceId","page","sinks"]},"Source":{"title":"Source","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"http":{"$ref":"#/definitions/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"projectId":{"type":"integer","example":123,"format":"int64"},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]},"version":{"$ref":"#/definitions/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}},"required":["projectId","branchId","sourceId","type","name","description","version","created"]},"SourcesList":{"title":"SourcesList","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"page":{"$ref":"#/definitions/PaginatedResponse"},"projectId":{"type":"integer","example":123,"format":"int64"},"sources":{"type":"array","items":{"$ref":"#/definitions/Source"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"required":["projectId","branchId","page","sources"]},"StreamCreateSinkRequestBody":{"title":"StreamCreateSinkRequestBody","type":"object","properties":{"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"table":{"$ref":"#/definitions/TableSinkCreate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["type","name"]},"StreamCreateSourceRequestBody":{"title":"StreamCreateSourceRequestBody","type":"object","properties":{"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]}},"example":{"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["type","name"]},"StreamUpdateSinkRequestBody":{"title":"StreamUpdateSinkRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"table":{"$ref":"#/definitions/TableSinkUpdate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}},"StreamUpdateSinkSettingsRequestBody":{"title":"StreamUpdateSinkSettingsRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"type":"array","items":{"$ref":"#/definitions/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]}},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}},"StreamUpdateSourceRequestBody":{"title":"StreamUpdateSourceRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"type":{"type":"string","example":"http","enum":["http"]}},"example":{"changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}},"StreamUpdateSourceSettingsRequestBody":{"title":"StreamUpdateSourceSettingsRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"type":"array","items":{"$ref":"#/definitions/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]}},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}},"TableColumn":{"title":"TableColumn","type":"object","properties":{"defaultValue":{"type":"string","description":"Fallback value if path doesn't exist.","example":"1"},"name":{"type":"string","description":"Column name.","example":"id-col"},"path":{"type":"string","description":"Path to the value.","example":"foo.bar[0]"},"rawString":{"type":"boolean","description":"Set to true if path value should use raw string instead of json-encoded value.","example":true},"template":{"$ref":"#/definitions/TableColumnTemplate"},"type":{"type":"string","description":"Column mapping type. This represents a static mapping (e.g. `body` or `headers`), or a custom mapping using a template language (`template`).","example":"body","enum":["uuid","datetime","ip","body","headers","path","template"]}},"description":"An output mapping defined by a template.","example":{"type":"body","name":"id-col","path":"foo.bar[0]","defaultValue":"1","rawString":true,"template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}},"required":["type","name"]},"TableColumnTemplate":{"title":"TableColumnTemplate","type":"object","properties":{"content":{"type":"string","example":"body.foo + \"-\" + body.bar","minLength":1,"maxLength":4096},"language":{"type":"string","example":"jsonnet","enum":["jsonnet"]}},"description":"Template column definition, for \"type\" = \"template\".","example":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"},"required":["language","content"]},"TableMapping":{"title":"TableMapping","type":"object","properties":{"columns":{"type":"array","items":{"$ref":"#/definitions/TableColumn"},"example":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}],"minItems":1,"maxItems":100}},"description":"Table mapping definition.","example":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]},"required":["columns"]},"TableSink":{"title":"TableSink","type":"object","properties":{"mapping":{"$ref":"#/definitions/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkCreate":{"title":"TableSinkCreate","type":"object","properties":{"mapping":{"$ref":"#/definitions/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkUpdate":{"title":"TableSinkUpdate","type":"object","properties":{"mapping":{"$ref":"#/definitions/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"Task":{"title":"Task","type":"object","properties":{"createdAt":{"type":"string","description":"Date and time of the task creation.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"duration":{"type":"integer","description":"Duration of the task in milliseconds.","example":123456789,"format":"int64"},"error":{"type":"string","example":"abc123"},"finishedAt":{"type":"string","description":"Date and time of the task end.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"isFinished":{"type":"boolean","description":"Shortcut for status != \"processing\".","example":false},"outputs":{"$ref":"#/definitions/TaskOutputs"},"result":{"type":"string","example":"abc123"},"status":{"type":"string","description":"Task status, one of: processing, success, error","example":"success","enum":["processing","success","error"]},"taskId":{"type":"string","example":"task_1234"},"type":{"type":"string","description":"Task type.","example":"abc123"},"url":{"type":"string","description":"URL of the task.","example":"abc123"}},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"required":["taskId","type","url","status","isFinished","createdAt"]},"TaskOutputs":{"title":"TaskOutputs","type":"object","properties":{"branchId":{"type":"integer","description":"ID of the parent branch.","example":345,"format":"int64"},"projectId":{"type":"integer","description":"ID of the parent project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"ID of the created/updated sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"ID of the created/updated source.","example":"github-webhook-source","minLength":1,"maxLength":48},"url":{"type":"string","description":"Absolute URL of the entity.","example":"abc123"}},"description":"Outputs generated by the task.","example":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"TestResult":{"title":"TestResult","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"projectId":{"type":"integer","example":123,"format":"int64"},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"tables":{"type":"array","items":{"$ref":"#/definitions/TestResultTable"},"description":"Table for each configured sink.","example":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]}},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","tables":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]},"required":["projectId","branchId","sourceId","tables"]},"TestResultColumn":{"title":"TestResultColumn","type":"object","properties":{"name":{"type":"string","description":"Column name.","example":"id"},"value":{"type":"string","description":"Column value.","example":"12345"}},"description":"Generated table column value, part of the test result.","example":{"name":"id","value":"12345"},"required":["name","value"]},"TestResultRow":{"title":"TestResultRow","type":"object","properties":{"columns":{"type":"array","items":{"$ref":"#/definitions/TestResultColumn"},"description":"Generated columns.","example":[{"name":"id","value":"12345"}]}},"description":"Generated table row, part of the test result.","example":{"columns":[{"name":"id","value":"12345"}]},"required":["columns"]},"TestResultTable":{"title":"TestResultTable","type":"object","properties":{"rows":{"type":"array","items":{"$ref":"#/definitions/TestResultRow"},"description":"Generated rows.","example":[{"columns":[{"name":"id","value":"12345"}]}]},"sinkId":{"type":"string","example":"github-pr-table-sink","minLength":1,"maxLength":48},"tableId":{"type":"string","example":"in.c-bucket.table"}},"description":"Generated table rows, part of the test result.","example":{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]},"required":["sinkId","tableId","rows"]},"Version":{"title":"Version","type":"object","properties":{"at":{"type":"string","description":"Date and time of the modification.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"},"description":{"type":"string","description":"Description of the change.","example":"The reason for the last change was..."},"hash":{"type":"string","description":"Hash of the entity state.","example":"f43e93acd97eceb3"},"number":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"description":"Version of the entity.","example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["number","hash","at","by","description"]}},"securityDefinitions":{"storage-api-token":{"type":"apiKey","description":"Storage Api Token Authentication.","name":"X-StorageApi-Token","in":"header"}}} \ No newline at end of file +{"swagger":"2.0","info":{"title":"Stream Service","description":"A service for continuously importing data to the Keboola platform.","license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0"},"version":"1.0"},"host":"stream.keboola.com","consumes":["application/json"],"produces":["application/json"],"paths":{"/":{"get":{"tags":["documentation"],"summary":"Redirect to /v1","description":"Redirect to /v1.","operationId":"ApiRootIndex","responses":{"301":{"description":"Moved Permanently response."}},"schemes":["https"]}},"/v1":{"get":{"tags":["documentation"],"summary":"List API name and link to documentation.","description":"List API name and link to documentation.","operationId":"ApiVersionIndex","responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/ServiceDetail","required":["api","documentation"]}}},"schemes":["https"]}},"/v1/branches/{branchId}/aggregation/sources":{"get":{"tags":["internal"],"summary":"Aggregation endpoint for sources","description":"Details about sources for the UI.","operationId":"AggregationSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/AggregatedSourcesResult","required":["projectId","branchId","page","sources"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources":{"get":{"tags":["configuration"],"summary":"List all sources","description":"List all sources in the branch.","operationId":"ListSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SourcesList","required":["projectId","branchId","page","sources"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"post":{"tags":["configuration"],"summary":"Create source","description":"Create a new source in the branch.","operationId":"CreateSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"CreateSourceRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamCreateSourceRequestBody","required":["type","name"]}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"409":{"description":"Conflict response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"422":{"description":"Unprocessable Entity response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/deleted":{"get":{"tags":["configuration"],"summary":"List all deleted sources","description":"List all deleted sources in the branch.","operationId":"ListDeletedSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SourcesList","required":["projectId","branchId","page","sources"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}":{"get":{"tags":["configuration"],"summary":"Get source","description":"Get the source definition.","operationId":"GetSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Source","required":["projectId","branchId","sourceId","type","name","description","version","created"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"delete":{"tags":["configuration"],"summary":"Delete source","description":"Delete the source.","operationId":"DeleteSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update source","description":"Update the source.","operationId":"UpdateSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"UpdateSourceRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSourceRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/disable":{"put":{"tags":["configuration"],"summary":"Disable source","description":"Disables the source.","operationId":"DisableSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/enable":{"put":{"tags":["configuration"],"summary":"Enable source","description":"Enables the source.","operationId":"EnableSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/settings":{"get":{"tags":["configuration"],"summary":"Get source settings","description":"Get source settings.","operationId":"GetSourceSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SettingsResult"}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update source settings","description":"Update source settings.","operationId":"UpdateSourceSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"UpdateSourceSettingsRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSourceSettingsRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks":{"get":{"tags":["configuration"],"summary":"List sinks","description":"List all sinks in the source.","operationId":"ListSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinksList","required":["projectId","branchId","sourceId","page","sinks"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"post":{"tags":["configuration"],"summary":"Create sink","description":"Create a new sink in the source.","operationId":"CreateSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"CreateSinkRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamCreateSinkRequestBody","required":["type","name"]}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"409":{"description":"Conflict response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"422":{"description":"Unprocessable Entity response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/deleted":{"get":{"tags":["configuration"],"summary":"List deleted sinks","description":"List all deleted sinks in the source.","operationId":"ListDeletedSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinksList","required":["projectId","branchId","sourceId","page","sinks"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}":{"get":{"tags":["configuration"],"summary":"Get sink","description":"Get the sink definition.","operationId":"GetSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Sink","required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"delete":{"tags":["configuration"],"summary":"Delete sink","description":"Delete the sink.","operationId":"DeleteSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update sink","description":"Update the sink.","operationId":"UpdateSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"UpdateSinkRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSinkRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/disable":{"put":{"tags":["configuration"],"summary":"Disable sink","description":"Disables the sink.","operationId":"DisableSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/enable":{"put":{"tags":["configuration"],"summary":"Enable sink","description":"Enables the sink.","operationId":"EnableSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/settings":{"get":{"tags":["configuration"],"summary":"Get sink settings","description":"Get the sink settings.","operationId":"GetSinkSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SettingsResult"}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]},"patch":{"tags":["configuration"],"summary":"Update sink settings","description":"Update sink settings.","operationId":"UpdateSinkSettings","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"UpdateSinkSettingsRequestBody","in":"body","required":true,"schema":{"$ref":"#/definitions/StreamUpdateSinkSettingsRequestBody"}}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear sink statistics","description":"Clears all statistics of the sink.","operationId":"SinkStatisticsClear","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response."},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/files":{"get":{"tags":["configuration"],"summary":"Sink files statistics","description":"Get files statistics of the sink.","operationId":"SinkStatisticsFiles","parameters":[{"name":"failedFiles","in":"query","description":"Filter for not imported files. If set to true, only not imported files will be included.","required":false,"type":"boolean","default":false},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinkStatisticsFilesResult","required":["files"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/total":{"get":{"tags":["configuration"],"summary":"Sink statistics total","description":"Get total statistics of the sink.","operationId":"SinkStatisticsTotal","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/SinkStatisticsTotalResult","required":["levels","total"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete sink","description":"Undelete the sink.","operationId":"UndeleteSink","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions":{"get":{"tags":["configuration"],"summary":"List sink versions","description":"List all sink versions.","operationId":"ListSinkVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/EntityVersions","required":["versions","page"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Sink version detail","description":"Sink version detail.","operationId":"SinkVersionDetail","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Version","required":["number","hash","at","by","description"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback sink version","description":"Rollback sink version.","operationId":"RollbackSinkVersion","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"sinkId","in":"path","description":"Unique ID of the sink.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear source statistics","description":"Clears all statistics of the source.","operationId":"SourceStatisticsClear","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response."},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/test":{"post":{"tags":["test"],"summary":"Test source payload mapping","description":"Tests configured mapping of the source and its sinks against an example request body.\n\nFor HTTP sources the body is the raw payload that an HTTP client would send (treated as a single record).\n\nFor OTLP sources the body must be a single **flattened OTLP record** — a JSON object with the same shape that the source produces internally for each log record, metric data point, or span. Attributes, resource, and scope are nested objects (not dotted keys); reference them in column mappings as `Body('attributes')['user.id']`, `Body('resource')['service.name']`, `Body('scope')['name']`, etc. Example flattened log record:\n```json\n{\n \"timestamp\": \"2024-01-15T10:30:00Z\",\n \"observed_timestamp\": \"2024-01-15T10:30:00Z\",\n \"severity_number\": 9,\n \"severity_text\": \"INFO\",\n \"body\": \"User logged in\",\n \"flags\": 0,\n \"attributes\": {\"user.id\": \"user-123\"},\n \"resource\": {\"service.name\": \"auth-service\"},\n \"scope\": {\"name\": \"github.com/my/auth\", \"version\": \"1.2.3\"}\n}\n```\nDo not send a raw OTLP protobuf or the multi-record envelope produced by an OTel SDK — the test endpoint intentionally evaluates one already-flattened record so the response is deterministic. For OTLP sources, the `signal` query parameter selects which signal type the request simulates for sink routing (`logs` by default); sinks whose `allowedSignals` filter rejects that signal are skipped in the result.","operationId":"TestSource","parameters":[{"name":"signal","in":"query","description":"OTLP signal type — one of logs, metrics, or traces.","required":false,"type":"string","enum":["logs","metrics","traces"]},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/TestResult","required":["projectId","branchId","sourceId","tables"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}},"422":{"description":"Unprocessable Entity response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete source","description":"Undelete the source.","operationId":"UndeleteSource","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions":{"get":{"tags":["configuration"],"summary":"List source versions","description":"List all source versions.","operationId":"ListSourceVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","required":false,"type":"string","default":""},{"name":"limit","in":"query","description":"Maximum number of returned records.","required":false,"type":"integer","default":100,"maximum":100,"minimum":1},{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/EntityVersions","required":["versions","page"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Source version detail","description":"Source version detail.","operationId":"SourceVersionDetail","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Version","required":["number","hash","at","by","description"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback source version","description":"Rollback source version.","operationId":"RollbackSourceVersion","parameters":[{"name":"branchId","in":"path","description":"ID of the branch or \"default\".","required":true,"type":"string"},{"name":"sourceId","in":"path","description":"Unique ID of the source.","required":true,"type":"string"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"type":"integer","minimum":1}],"responses":{"202":{"description":"Accepted response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}},"/v1/documentation/openapi.json":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 JSON Specification","operationId":"OpenapiJson","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/documentation/openapi.yaml":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 YAML Specification","operationId":"OpenapiYaml","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/documentation/openapi3.json":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 JSON Specification","operationId":"Openapi3Json","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/documentation/openapi3.yaml":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 YAML Specification","operationId":"Openapi3Yaml","responses":{"200":{"description":"File downloaded","schema":{"type":"file"}}},"schemes":["http","https"]}},"/v1/tasks/{taskId}":{"get":{"tags":["configuration"],"summary":"Get task","description":"Get details of a task.","operationId":"GetTask","parameters":[{"name":"taskId","in":"path","description":"Unique ID of the task.","required":true,"type":"string"}],"responses":{"200":{"description":"OK response.","schema":{"$ref":"#/definitions/Task","required":["taskId","type","url","status","isFinished","createdAt"]}},"404":{"description":"Not Found response.","schema":{"$ref":"#/definitions/GenericError","required":["statusCode","error","message"]}}},"schemes":["https"],"security":[{"storage-api-token":null}]}}},"definitions":{"AggregatedSink":{"title":"AggregatedSink","type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","example":123,"format":"int64"},"sinkId":{"type":"string","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"statistics":{"$ref":"#/definitions/AggregatedStatistics"},"table":{"$ref":"#/definitions/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/definitions/Version"}},"description":"A mapping from imported data to a destination table.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"AggregatedSource":{"title":"AggregatedSource","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"http":{"$ref":"#/definitions/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"otlp":{"$ref":"#/definitions/OTLPSource"},"projectId":{"type":"integer","example":123,"format":"int64"},"sinks":{"type":"array","items":{"$ref":"#/definitions/AggregatedSink"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]},"version":{"$ref":"#/definitions/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},"required":["projectId","branchId","sourceId","type","name","description","version","created","sinks"]},"AggregatedSourcesResult":{"title":"AggregatedSourcesResult","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"page":{"$ref":"#/definitions/PaginatedResponse"},"projectId":{"type":"integer","example":123,"format":"int64"},"sources":{"type":"array","items":{"$ref":"#/definitions/AggregatedSource"},"example":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sinks":[],"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]}},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sinks":[],"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]},"required":["projectId","branchId","page","sources"]},"AggregatedStatistics":{"title":"AggregatedStatistics","type":"object","properties":{"files":{"type":"array","items":{"$ref":"#/definitions/SinkFile"},"example":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"levels":{"$ref":"#/definitions/Levels"},"total":{"$ref":"#/definitions/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["total","levels","files"]},"By":{"title":"By","type":"object","properties":{"tokenDesc":{"type":"string","description":"Description of the token.","example":"john.green@company.com"},"tokenId":{"type":"string","description":"ID of the token.","example":"896455"},"type":{"type":"string","description":"Date and time of deletion.","example":"user","enum":["system","user"]},"userId":{"type":"string","description":"ID of the user.","example":"578621"},"userName":{"type":"string","description":"Name of the user.","example":"John Green"}},"description":"Information about the operation actor.","example":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"required":["type"]},"CreatedEntity":{"title":"CreatedEntity","type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"}},"description":"Information about the entity creation.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DeletedEntity":{"title":"DeletedEntity","type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"}},"description":"Information about the deleted entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DisabledEntity":{"title":"DisabledEntity","type":"object","properties":{"at":{"type":"string","description":"Date and time of disabling.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"},"reason":{"type":"string","description":"Why was the entity disabled?","example":"Disabled for recurring problems."}},"description":"Information about the disabled entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"required":["at","by","reason"]},"EntityVersions":{"title":"EntityVersions","type":"object","properties":{"page":{"$ref":"#/definitions/PaginatedResponse"},"versions":{"type":"array","items":{"$ref":"#/definitions/Version"},"example":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}]}},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}},"required":["versions","page"]},"GenericError":{"title":"GenericError","type":"object","properties":{"error":{"type":"string","description":"Name of error.","example":"stream.internalError"},"message":{"type":"string","description":"Error message.","example":"Internal Error"},"statusCode":{"type":"integer","description":"HTTP status code.","example":500,"format":"int64"}},"description":"Source already exists in the branch.","example":{"statusCode":409,"error":"stream.api.sourceAlreadyExists","message":"Source already exists in the branch."},"required":["statusCode","error","message"]},"HTTPSource":{"title":"HTTPSource","type":"object","properties":{"url":{"type":"string","description":"URL of the HTTP source. Contains secret used for authentication.","example":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"}},"description":"HTTP source details for \"type\" = \"http\".","example":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"required":["url"]},"Level":{"title":"Level","type":"object","properties":{"compressedSize":{"type":"integer","description":"Compressed size of data in bytes.","example":1,"format":"int64"},"firstRecordAt":{"type":"string","description":"Timestamp of the first received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"lastRecordAt":{"type":"string","description":"Timestamp of the last received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"recordsCount":{"type":"integer","example":1,"format":"int64"},"uncompressedSize":{"type":"integer","description":"Uncompressed size of data in bytes.","example":1,"format":"int64"}},"example":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"required":["recordsCount","compressedSize","uncompressedSize"]},"Levels":{"title":"Levels","type":"object","properties":{"local":{"$ref":"#/definitions/Level"},"staging":{"$ref":"#/definitions/Level"},"target":{"$ref":"#/definitions/Level"}},"example":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"OTLPSource":{"title":"OTLPSource","type":"object","properties":{"baseUrl":{"type":"string","description":"Endpoint URL without the secret. Use this together with the `secret` field via the `Authorization: Bearer \u003csecret\u003e` header so the secret stays out of access/CDN/APM logs. The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.","example":"https://stream-in.keboola.com/otlp/123/my-source"},"secret":{"type":"string","description":"48-character secret authenticating writes to this source. Send it as `Authorization: Bearer \u003csecret\u003e` to the `baseUrl`.","example":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"url":{"type":"string","description":"Endpoint URL with the secret embedded as the last path segment. Convenient for SDKs that authenticate by URL only. The OpenTelemetry SDK automatically appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — do not append a signal path yourself. Most SDK exporters reject or silently strip the suffix.","example":"https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"}},"description":"OTLP/HTTP source details for \"type\" = \"otlp\".","example":{"url":"https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","baseUrl":"https://stream-in.keboola.com/otlp/123/my-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"required":["url","baseUrl","secret"]},"PaginatedResponse":{"title":"PaginatedResponse","type":"object","properties":{"afterId":{"type":"string","description":"Current offset.","example":"my-object-123"},"lastId":{"type":"string","description":"ID of the last record in the response.","example":"my-object-456"},"limit":{"type":"integer","description":"Current limit.","example":100,"format":"int64"},"totalCount":{"type":"integer","description":"Total count of all records.","example":1000,"format":"int64"}},"example":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"required":["afterId","limit","lastId","totalCount"]},"ServiceDetail":{"title":"ServiceDetail","type":"object","properties":{"api":{"type":"string","description":"Name of the API","example":"stream"},"documentation":{"type":"string","description":"URL of the API documentation.","example":"https://stream.keboola.com/v1/documentation"}},"example":{"api":"stream","documentation":"https://stream.keboola.com/v1/documentation"},"required":["api","documentation"]},"SettingPatch":{"title":"SettingPatch","type":"object","properties":{"key":{"type":"string","description":"Key path.","example":"some.service.limit","minLength":1},"value":{"description":"A new key value. Use null to reset the value to the default value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","value":"1m20s"},"required":["key"]},"SettingResult":{"title":"SettingResult","type":"object","properties":{"defaultValue":{"description":"Default value.","example":"30s"},"description":{"type":"string","description":"Key description.","example":"Minimal interval between uploads."},"key":{"type":"string","description":"Key path.","example":"some.service.limit"},"overwritten":{"type":"boolean","description":"True, if the default value is locally overwritten.","example":true},"protected":{"type":"boolean","description":"True, if only a super admin can modify the key.","example":false},"type":{"type":"string","description":"Value type.","example":"string","enum":["string","int","float","bool","[]string","[]int","[]float"]},"validation":{"type":"string","description":"Validation rules as a string definition.","example":"minDuration=15s"},"value":{"description":"Actual value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"},"required":["key","type","description","value","defaultValue","overwritten","protected"]},"SettingsResult":{"title":"SettingsResult","type":"object","properties":{"settings":{"type":"array","items":{"$ref":"#/definitions/SettingResult"},"example":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"Sink":{"title":"Sink","type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","example":123,"format":"int64"},"sinkId":{"type":"string","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/definitions/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/definitions/Version"}},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"SinkFile":{"title":"SinkFile","type":"object","properties":{"closingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"openedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAfter":{"type":"string","description":"Next attempt time.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAttempt":{"type":"integer","description":"Number of failed attempts.","example":3,"format":"int64"},"retryReason":{"type":"string","description":"Reason of the last failed attempt.","example":"network problem"},"state":{"type":"string","example":"writing","enum":["writing","closing","importing","imported"]},"statistics":{"$ref":"#/definitions/SinkFileStatistics"}},"example":{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}},"required":["state","openedAt"]},"SinkFileStatistics":{"title":"SinkFileStatistics","type":"object","properties":{"levels":{"$ref":"#/definitions/Levels"},"total":{"$ref":"#/definitions/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["total","levels"]},"SinkStatisticsFilesResult":{"title":"SinkStatisticsFilesResult","type":"object","properties":{"files":{"type":"array","items":{"$ref":"#/definitions/SinkFile"},"example":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}},"example":{"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["files"]},"SinkStatisticsTotalResult":{"title":"SinkStatisticsTotalResult","type":"object","properties":{"levels":{"$ref":"#/definitions/Levels"},"total":{"$ref":"#/definitions/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["levels","total"]},"SinksList":{"title":"SinksList","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"page":{"$ref":"#/definitions/PaginatedResponse"},"projectId":{"type":"integer","example":123,"format":"int64"},"sinks":{"type":"array","items":{"$ref":"#/definitions/Sink"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"required":["projectId","branchId","sourceId","page","sinks"]},"Source":{"title":"Source","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"created":{"$ref":"#/definitions/CreatedEntity"},"deleted":{"$ref":"#/definitions/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/definitions/DisabledEntity"},"http":{"$ref":"#/definitions/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"otlp":{"$ref":"#/definitions/OTLPSource"},"projectId":{"type":"integer","example":123,"format":"int64"},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]},"version":{"$ref":"#/definitions/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},"required":["projectId","branchId","sourceId","type","name","description","version","created"]},"SourcesList":{"title":"SourcesList","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"page":{"$ref":"#/definitions/PaginatedResponse"},"projectId":{"type":"integer","example":123,"format":"int64"},"sources":{"type":"array","items":{"$ref":"#/definitions/Source"},"example":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]}},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]},"required":["projectId","branchId","page","sources"]},"StreamCreateSinkRequestBody":{"title":"StreamCreateSinkRequestBody","type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"table":{"$ref":"#/definitions/TableSinkCreate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["type","name"]},"StreamCreateSourceRequestBody":{"title":"StreamCreateSourceRequestBody","type":"object","properties":{"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]}},"example":{"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["type","name"]},"StreamUpdateSinkRequestBody":{"title":"StreamUpdateSinkRequestBody","type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"table":{"$ref":"#/definitions/TableSinkUpdate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}},"StreamUpdateSinkSettingsRequestBody":{"title":"StreamUpdateSinkSettingsRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"type":"array","items":{"$ref":"#/definitions/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]}},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}},"StreamUpdateSourceRequestBody":{"title":"StreamUpdateSourceRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"type":{"type":"string","example":"http","enum":["http","otlp"]}},"example":{"changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}},"StreamUpdateSourceSettingsRequestBody":{"title":"StreamUpdateSourceSettingsRequestBody","type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"type":"array","items":{"$ref":"#/definitions/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]}},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}},"TableColumn":{"title":"TableColumn","type":"object","properties":{"defaultValue":{"type":"string","description":"Fallback value if path doesn't exist.","example":"1"},"name":{"type":"string","description":"Column name.","example":"id-col"},"path":{"type":"string","description":"Path to the value.","example":"foo.bar[0]"},"rawString":{"type":"boolean","description":"Set to true if path value should use raw string instead of json-encoded value.","example":true},"template":{"$ref":"#/definitions/TableColumnTemplate"},"type":{"type":"string","description":"Column mapping type. This represents a static mapping (e.g. `body` or `headers`), or a custom mapping using a template language (`template`).","example":"body","enum":["uuid","datetime","ip","body","headers","path","template"]}},"description":"An output mapping defined by a template.","example":{"type":"body","name":"id-col","path":"foo.bar[0]","defaultValue":"1","rawString":true,"template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}},"required":["type","name"]},"TableColumnTemplate":{"title":"TableColumnTemplate","type":"object","properties":{"content":{"type":"string","example":"body.foo + \"-\" + body.bar","minLength":1,"maxLength":4096},"language":{"type":"string","example":"jsonnet","enum":["jsonnet"]}},"description":"Template column definition, for \"type\" = \"template\".","example":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"},"required":["language","content"]},"TableMapping":{"title":"TableMapping","type":"object","properties":{"columns":{"type":"array","items":{"$ref":"#/definitions/TableColumn"},"example":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}],"minItems":1,"maxItems":100}},"description":"Table mapping definition.","example":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]},"required":["columns"]},"TableSink":{"title":"TableSink","type":"object","properties":{"mapping":{"$ref":"#/definitions/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkCreate":{"title":"TableSinkCreate","type":"object","properties":{"mapping":{"$ref":"#/definitions/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkUpdate":{"title":"TableSinkUpdate","type":"object","properties":{"mapping":{"$ref":"#/definitions/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"Task":{"title":"Task","type":"object","properties":{"createdAt":{"type":"string","description":"Date and time of the task creation.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"duration":{"type":"integer","description":"Duration of the task in milliseconds.","example":123456789,"format":"int64"},"error":{"type":"string","example":"abc123"},"finishedAt":{"type":"string","description":"Date and time of the task end.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"isFinished":{"type":"boolean","description":"Shortcut for status != \"processing\".","example":false},"outputs":{"$ref":"#/definitions/TaskOutputs"},"result":{"type":"string","example":"abc123"},"status":{"type":"string","description":"Task status, one of: processing, success, error","example":"success","enum":["processing","success","error"]},"taskId":{"type":"string","example":"task_1234"},"type":{"type":"string","description":"Task type.","example":"abc123"},"url":{"type":"string","description":"URL of the task.","example":"abc123"}},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"required":["taskId","type","url","status","isFinished","createdAt"]},"TaskOutputs":{"title":"TaskOutputs","type":"object","properties":{"branchId":{"type":"integer","description":"ID of the parent branch.","example":345,"format":"int64"},"projectId":{"type":"integer","description":"ID of the parent project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"ID of the created/updated sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"ID of the created/updated source.","example":"github-webhook-source","minLength":1,"maxLength":48},"url":{"type":"string","description":"Absolute URL of the entity.","example":"abc123"}},"description":"Outputs generated by the task.","example":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"TestResult":{"title":"TestResult","type":"object","properties":{"branchId":{"type":"integer","example":345,"format":"int64"},"projectId":{"type":"integer","example":123,"format":"int64"},"sourceId":{"type":"string","example":"github-webhook-source","minLength":1,"maxLength":48},"tables":{"type":"array","items":{"$ref":"#/definitions/TestResultTable"},"description":"Table for each configured sink.","example":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]}},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","tables":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]},"required":["projectId","branchId","sourceId","tables"]},"TestResultColumn":{"title":"TestResultColumn","type":"object","properties":{"name":{"type":"string","description":"Column name.","example":"id"},"value":{"type":"string","description":"Column value.","example":"12345"}},"description":"Generated table column value, part of the test result.","example":{"name":"id","value":"12345"},"required":["name","value"]},"TestResultRow":{"title":"TestResultRow","type":"object","properties":{"columns":{"type":"array","items":{"$ref":"#/definitions/TestResultColumn"},"description":"Generated columns.","example":[{"name":"id","value":"12345"}]}},"description":"Generated table row, part of the test result.","example":{"columns":[{"name":"id","value":"12345"}]},"required":["columns"]},"TestResultTable":{"title":"TestResultTable","type":"object","properties":{"rows":{"type":"array","items":{"$ref":"#/definitions/TestResultRow"},"description":"Generated rows.","example":[{"columns":[{"name":"id","value":"12345"}]}]},"sinkId":{"type":"string","example":"github-pr-table-sink","minLength":1,"maxLength":48},"tableId":{"type":"string","example":"in.c-bucket.table"}},"description":"Generated table rows, part of the test result.","example":{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]},"required":["sinkId","tableId","rows"]},"Version":{"title":"Version","type":"object","properties":{"at":{"type":"string","description":"Date and time of the modification.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/definitions/By"},"description":{"type":"string","description":"Description of the change.","example":"The reason for the last change was..."},"hash":{"type":"string","description":"Hash of the entity state.","example":"f43e93acd97eceb3"},"number":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"description":"Version of the entity.","example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["number","hash","at","by","description"]}},"securityDefinitions":{"storage-api-token":{"type":"apiKey","description":"Storage Api Token Authentication.","name":"X-StorageApi-Token","in":"header"}}} \ No newline at end of file diff --git a/internal/pkg/service/stream/api/openapi/openapi.yaml b/internal/pkg/service/stream/api/openapi/openapi.yaml index dce0915dde..596397b46a 100644 --- a/internal/pkg/service/stream/api/openapi/openapi.yaml +++ b/internal/pkg/service/stream/api/openapi/openapi.yaml @@ -1354,9 +1354,37 @@ paths: tags: - test summary: Test source payload mapping - description: Tests configured mapping of the source and its sinks. + description: |- + Tests configured mapping of the source and its sinks against an example request body. + + For HTTP sources the body is the raw payload that an HTTP client would send (treated as a single record). + + For OTLP sources the body must be a single **flattened OTLP record** — a JSON object with the same shape that the source produces internally for each log record, metric data point, or span. Attributes, resource, and scope are nested objects (not dotted keys); reference them in column mappings as `Body('attributes')['user.id']`, `Body('resource')['service.name']`, `Body('scope')['name']`, etc. Example flattened log record: + ```json + { + "timestamp": "2024-01-15T10:30:00Z", + "observed_timestamp": "2024-01-15T10:30:00Z", + "severity_number": 9, + "severity_text": "INFO", + "body": "User logged in", + "flags": 0, + "attributes": {"user.id": "user-123"}, + "resource": {"service.name": "auth-service"}, + "scope": {"name": "github.com/my/auth", "version": "1.2.3"} + } + ``` + Do not send a raw OTLP protobuf or the multi-record envelope produced by an OTel SDK — the test endpoint intentionally evaluates one already-flattened record so the response is deterministic. For OTLP sources, the `signal` query parameter selects which signal type the request simulates for sink routing (`logs` by default); sinks whose `allowedSignals` filter rejects that signal are skipped in the result. operationId: TestSource parameters: + - name: signal + in: query + description: OTLP signal type — one of logs, metrics, or traces. + required: false + type: string + enum: + - logs + - metrics + - traces - name: branchId in: path description: ID of the branch or "default". @@ -1725,6 +1753,18 @@ definitions: title: AggregatedSink type: object properties: + allowedSignals: + type: array + items: + type: string + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs branchId: type: integer example: 345 @@ -1780,6 +1820,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -1939,6 +1981,8 @@ definitions: example: Github Webhook Source minLength: 1 maxLength: 40 + otlp: + $ref: '#/definitions/OTLPSource' projectId: type: integer example: 123 @@ -1955,6 +1999,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -2087,184 +2133,41 @@ definitions: example: http enum: - http + - otlp version: $ref: '#/definitions/Version' description: Source of data for further processing, start of the stream, max 100 sources per a branch. example: - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green + branchId: 5678 created: - at: "2022-04-28T14:20:04.000Z" + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user tokenId: "896455" - tokenDesc: john.green@company.com + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -2294,363 +2197,131 @@ definitions: items: $ref: '#/definitions/AggregatedSource' example: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green + - branchId: 5678 created: - at: "2022-04-28T14:20:04.000Z" + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: type: user - tokenId: "896455" - tokenDesc: john.green@company.com userId: "578621" userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - example: - projectId: 123 - branchId: 345 - page: - limit: 100 - totalCount: 1000 - afterId: my-object-123 - lastId: my-object-456 - sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sinks: [] + sourceId: my-http-source + type: http version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user tokenId: "896455" - tokenDesc: john.green@company.com + type: user userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 + description: New source. + hash: f43e93acd97eceb3 + number: 1 + example: + projectId: 123 + branchId: 345 + page: + limit: 100 + totalCount: 1000 + afterId: my-object-123 + lastId: my-object-456 + sources: + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sinks: [] + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -2953,10 +2624,10 @@ definitions: url: type: string description: URL of the HTTP source. Contains secret used for authentication. - example: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb + example: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX description: HTTP source details for "type" = "http". example: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX required: - url Level: @@ -3026,6 +2697,31 @@ definitions: recordsCount: 1 compressedSize: 1 uncompressedSize: 1 + OTLPSource: + title: OTLPSource + type: object + properties: + baseUrl: + type: string + description: 'Endpoint URL without the secret. Use this together with the `secret` field via the `Authorization: Bearer ` header so the secret stays out of access/CDN/APM logs. The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.' + example: https://stream-in.keboola.com/otlp/123/my-source + secret: + type: string + description: '48-character secret authenticating writes to this source. Send it as `Authorization: Bearer ` to the `baseUrl`.' + example: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: + type: string + description: Endpoint URL with the secret embedded as the last path segment. Convenient for SDKs that authenticate by URL only. The OpenTelemetry SDK automatically appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — do not append a signal path yourself. Most SDK exporters reject or silently strip the suffix. + example: https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + description: OTLP/HTTP source details for "type" = "otlp". + example: + url: https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + baseUrl: https://stream-in.keboola.com/otlp/123/my-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + required: + - url + - baseUrl + - secret PaginatedResponse: title: PaginatedResponse type: object @@ -3185,6 +2881,18 @@ definitions: title: Sink type: object properties: + allowedSignals: + type: array + items: + type: string + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs branchId: type: integer example: 345 @@ -3237,6 +2945,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -3571,6 +3281,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -3654,6 +3366,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -3748,6 +3462,8 @@ definitions: example: Github Webhook Source minLength: 1 maxLength: 40 + otlp: + $ref: '#/definitions/OTLPSource' projectId: type: integer example: 123 @@ -3762,54 +3478,40 @@ definitions: example: http enum: - http + - otlp version: $ref: '#/definitions/Version' description: Source of data for further processing, start of the stream, max 100 sources per a branch. example: - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green + branchId: 5678 created: - at: "2022-04-28T14:20:04.000Z" + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user tokenId: "896455" - tokenDesc: john.green@company.com + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -3838,50 +3540,62 @@ definitions: items: $ref: '#/definitions/Source' example: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 example: projectId: 123 branchId: 345 @@ -3891,50 +3605,62 @@ definitions: afterId: my-object-123 lastId: my-object-456 sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -3944,6 +3670,18 @@ definitions: title: StreamCreateSinkRequestBody type: object properties: + allowedSignals: + type: array + items: + type: string + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs description: type: string description: Description of the source. @@ -3973,6 +3711,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -4027,6 +3767,7 @@ definitions: example: http enum: - http + - otlp example: sourceId: github-webhook-source type: http @@ -4039,6 +3780,18 @@ definitions: title: StreamUpdateSinkRequestBody type: object properties: + allowedSignals: + type: array + items: + type: string + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs changeDescription: type: string description: Description of the modification, description of the version. @@ -4066,6 +3819,8 @@ definitions: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -4135,6 +3890,7 @@ definitions: example: http enum: - http + - otlp example: changeDescription: Renamed. type: http diff --git a/internal/pkg/service/stream/api/openapi/openapi3.json b/internal/pkg/service/stream/api/openapi/openapi3.json index 8271471d2b..b641859e52 100644 --- a/internal/pkg/service/stream/api/openapi/openapi3.json +++ b/internal/pkg/service/stream/api/openapi/openapi3.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"title":"Stream Service","description":"A service for continuously importing data to the Keboola platform.","license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0"},"version":"1.0"},"servers":[{"url":"https://stream.{stack}","variables":{"stack":{"enum":["keboola.com","eu-central-1.keboola.com","north-europe.azure.keboola.com","eu-west-1.aws.keboola.dev","east-us-2.azure.keboola-testing.com"],"default":"keboola.com"}}},{"url":"http://localhost:8001"}],"paths":{"/":{"get":{"tags":["documentation"],"summary":"Redirect to /v1","description":"Redirect to /v1.","operationId":"ApiRootIndex","responses":{"301":{"description":"Moved Permanently response."}}}},"/v1":{"get":{"tags":["documentation"],"summary":"List API name and link to documentation.","description":"List API name and link to documentation.","operationId":"ApiVersionIndex","responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ServiceDetail"},"example":{"api":"stream","documentation":"https://stream.keboola.com/v1/documentation"}}}}}}},"/v1/branches/{branchId}/aggregation/sources":{"get":{"tags":["internal"],"summary":"Aggregation endpoint for sources","description":"Details about sources for the UI.","operationId":"AggregationSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AggregatedSourcesResult"},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]}]}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources":{"get":{"tags":["configuration"],"summary":"List all sources","description":"List all sources in the branch.","operationId":"ListSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SourcesList"},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}}}}},"security":[{"storage-api-token":[]}]},"post":{"tags":["configuration"],"summary":"Create source","description":"Create a new source in the branch.","operationId":"CreateSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSourceRequestBody"},"example":{"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"409":{"description":"stream.api.sourceAlreadyExists: Source already exists in the branch.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":409,"error":"stream.api.sourceAlreadyExists","message":"Source already exists in the branch."}}}},"422":{"description":"stream.api.resourceLimitReached: Resource limit reached.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":422,"error":"stream.api.resourceLimitReached","message":"Maximum number of sources per project is 100."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/deleted":{"get":{"tags":["configuration"],"summary":"List all deleted sources","description":"List all deleted sources in the branch.","operationId":"ListDeletedSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SourcesList"},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}":{"delete":{"tags":["configuration"],"summary":"Delete source","description":"Delete the source.","operationId":"DeleteSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"get":{"tags":["configuration"],"summary":"Get source","description":"Get the source definition.","operationId":"GetSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Source"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update source","description":"Update the source.","operationId":"UpdateSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSourceRequestBody"},"example":{"changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/disable":{"put":{"tags":["configuration"],"summary":"Disable source","description":"Disables the source.","operationId":"DisableSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/enable":{"put":{"tags":["configuration"],"summary":"Enable source","description":"Enables the source.","operationId":"EnableSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/settings":{"get":{"tags":["configuration"],"summary":"Get source settings","description":"Get source settings.","operationId":"GetSourceSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SettingsResult"},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update source settings","description":"Update source settings.","operationId":"UpdateSourceSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSourceSettingsRequestBody"},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.forbidden: Modification of protected settings is forbidden.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.forbidden","message":"Cannot modify protected keys: \"storage.level.local.encoding.compression.gzip.blockSize\"."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks":{"get":{"tags":["configuration"],"summary":"List sinks","description":"List all sinks in the source.","operationId":"ListSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinksList"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"post":{"tags":["configuration"],"summary":"Create sink","description":"Create a new sink in the source.","operationId":"CreateSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSinkRequestBody"},"example":{"sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}},"409":{"description":"stream.api.sinkAlreadyExists: Sink already exists in the source.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":409,"error":"stream.api.sinkAlreadyExists","message":"Sink already exists in the source."}}}},"422":{"description":"stream.api.resourceLimitReached: Resource limit reached.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":422,"error":"stream.api.resourceLimitReached","message":"Maximum number of sources per project is 100."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/deleted":{"get":{"tags":["configuration"],"summary":"List deleted sinks","description":"List all deleted sinks in the source.","operationId":"ListDeletedSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinksList"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}":{"delete":{"tags":["configuration"],"summary":"Delete sink","description":"Delete the sink.","operationId":"DeleteSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]},"get":{"tags":["configuration"],"summary":"Get sink","description":"Get the sink definition.","operationId":"GetSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Sink"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update sink","description":"Update the sink.","operationId":"UpdateSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSinkRequestBody"},"example":{"changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/disable":{"put":{"tags":["configuration"],"summary":"Disable sink","description":"Disables the sink.","operationId":"DisableSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/enable":{"put":{"tags":["configuration"],"summary":"Enable sink","description":"Enables the sink.","operationId":"EnableSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/settings":{"get":{"tags":["configuration"],"summary":"Get sink settings","description":"Get the sink settings.","operationId":"GetSinkSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SettingsResult"},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update sink settings","description":"Update sink settings.","operationId":"UpdateSinkSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSourceSettingsRequestBody"},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.forbidden: Modification of protected settings is forbidden.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.forbidden","message":"Cannot modify protected keys: \"storage.level.local.encoding.compression.gzip.blockSize\"."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear sink statistics","description":"Clears all statistics of the sink.","operationId":"SinkStatisticsClear","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response."},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/files":{"get":{"tags":["configuration"],"summary":"Sink files statistics","description":"Get files statistics of the sink.","operationId":"SinkStatisticsFiles","parameters":[{"name":"failedFiles","in":"query","description":"Filter for not imported files. If set to true, only not imported files will be included.","allowEmptyValue":true,"schema":{"type":"boolean","description":"Filter for not imported files. If set to true, only not imported files will be included.","default":false,"example":false},"example":false},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinkStatisticsFilesResult"},"example":{"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/total":{"get":{"tags":["configuration"],"summary":"Sink statistics total","description":"Get total statistics of the sink.","operationId":"SinkStatisticsTotal","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinkStatisticsTotalResult"},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete sink","description":"Undelete the sink.","operationId":"UndeleteSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions":{"get":{"tags":["configuration"],"summary":"List sink versions","description":"List all sink versions.","operationId":"ListSinkVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EntityVersions"},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Sink version detail","description":"Sink version detail.","operationId":"SinkVersionDetail","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Version"},"example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback sink version","description":"Rollback sink version.","operationId":"RollbackSinkVersion","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear source statistics","description":"Clears all statistics of the source.","operationId":"SourceStatisticsClear","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response."},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/test":{"post":{"tags":["test"],"summary":"Test source payload mapping","description":"Tests configured mapping of the source and its sinks.","operationId":"TestSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestResult"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","tables":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}},"422":{"description":"stream.api.invalidColumnValue: Invalid data for sink.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":422,"error":"stream.api.invalidColumnValue","message":"Invalid value for column \"name\": path \"name\" not found in the body"}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete source","description":"Undelete the source.","operationId":"UndeleteSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions":{"get":{"tags":["configuration"],"summary":"List source versions","description":"List all source versions.","operationId":"ListSourceVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EntityVersions"},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Source version detail","description":"Source version detail.","operationId":"SourceVersionDetail","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Version"},"example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback source version","description":"Rollback source version.","operationId":"RollbackSourceVersion","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/documentation/openapi.json":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 JSON Specification","operationId":"OpenapiJson","responses":{"200":{"description":"File downloaded"}}}},"/v1/documentation/openapi.yaml":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 YAML Specification","operationId":"OpenapiYaml","responses":{"200":{"description":"File downloaded"}}}},"/v1/documentation/openapi3.json":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 JSON Specification","operationId":"Openapi3Json","responses":{"200":{"description":"File downloaded"}}}},"/v1/documentation/openapi3.yaml":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 YAML Specification","operationId":"Openapi3Yaml","responses":{"200":{"description":"File downloaded"}}}},"/v1/tasks/{taskId}":{"get":{"tags":["configuration"],"summary":"Get task","description":"Get details of a task.","operationId":"GetTask","parameters":[{"name":"taskId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the task.","example":"task_1234"},"example":"task_1234"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.taskNotFound: Task not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.taskNotFound","message":"Task \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}}},"components":{"schemas":{"AggregatedSink":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"statistics":{"$ref":"#/components/schemas/AggregatedStatistics"},"table":{"$ref":"#/components/schemas/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"A mapping from imported data to a destination table.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"AggregatedSinks":{"type":"array","items":{"$ref":"#/components/schemas/AggregatedSink"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]},"AggregatedSource":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"http":{"$ref":"#/components/schemas/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinks":{"$ref":"#/components/schemas/AggregatedSinks"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]},"required":["projectId","branchId","sourceId","type","name","description","version","created","sinks"]},"AggregatedSources":{"type":"array","items":{"$ref":"#/components/schemas/AggregatedSource"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]}]},"AggregatedSourcesRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100}},"example":{"branchId":"default","afterId":"my-object-123","limit":100},"required":["branchId"]},"AggregatedSourcesResult":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"page":{"$ref":"#/components/schemas/PaginatedResponse"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sources":{"$ref":"#/components/schemas/AggregatedSources"}},"description":"List of sources, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]}]},"required":["projectId","branchId","page","sources"]},"AggregatedStatistics":{"type":"object","properties":{"files":{"$ref":"#/components/schemas/SinkFiles"},"levels":{"$ref":"#/components/schemas/Levels"},"total":{"$ref":"#/components/schemas/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["total","levels","files"]},"By":{"type":"object","properties":{"tokenDesc":{"type":"string","description":"Description of the token.","example":"john.green@company.com"},"tokenId":{"type":"string","description":"ID of the token.","example":"896455"},"type":{"type":"string","description":"Date and time of deletion.","example":"user","enum":["system","user"]},"userId":{"type":"string","description":"ID of the user.","example":"578621"},"userName":{"type":"string","description":"Name of the user.","example":"John Green"}},"description":"Information about the operation actor.","example":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"required":["type"]},"CreateSinkRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSinkCreate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["branchId","sourceId","type","name"]},"CreateSinkRequestBody":{"type":"object","properties":{"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSinkCreate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["type","name"]},"CreateSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["branchId","type","name"]},"CreateSourceRequestBody":{"type":"object","properties":{"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]}},"example":{"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["type","name"]},"CreatedEntity":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"}},"description":"Information about the entity creation.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DeletedEntity":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"}},"description":"Information about the deleted entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DisabledEntity":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of disabling.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"},"reason":{"type":"string","description":"Why was the entity disabled?","example":"Disabled for recurring problems."}},"description":"Information about the disabled entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"required":["at","by","reason"]},"EntityVersions":{"type":"object","properties":{"page":{"$ref":"#/components/schemas/PaginatedResponse"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/Version"},"example":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}]}},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}},"required":["versions","page"]},"GenericError":{"type":"object","properties":{"error":{"type":"string","description":"Name of error.","example":"stream.internalError"},"message":{"type":"string","description":"Error message.","example":"Internal Error"},"statusCode":{"type":"integer","description":"HTTP status code.","example":500,"format":"int64"}},"description":"Generic error.","example":{"statusCode":500,"error":"stream.internalError","message":"Internal Error"},"required":["statusCode","error","message"]},"GetSinkRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"},"required":["branchId","sourceId","sinkId"]},"GetSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source"},"required":["branchId","sourceId"]},"GetTaskRequest":{"type":"object","properties":{"taskId":{"type":"string","description":"Unique ID of the task.","example":"task_1234"}},"example":{"taskId":"task_1234"},"required":["taskId"]},"HTTPSource":{"type":"object","properties":{"url":{"type":"string","description":"URL of the HTTP source. Contains secret used for authentication.","example":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"}},"description":"HTTP source details for \"type\" = \"http\".","example":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"required":["url"]},"Level":{"type":"object","properties":{"compressedSize":{"type":"integer","description":"Compressed size of data in bytes.","example":1,"format":"int64"},"firstRecordAt":{"type":"string","description":"Timestamp of the first received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"lastRecordAt":{"type":"string","description":"Timestamp of the last received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"recordsCount":{"type":"integer","example":1,"format":"int64"},"uncompressedSize":{"type":"integer","description":"Uncompressed size of data in bytes.","example":1,"format":"int64"}},"example":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"required":["recordsCount","compressedSize","uncompressedSize"]},"Levels":{"type":"object","properties":{"local":{"$ref":"#/components/schemas/Level"},"staging":{"$ref":"#/components/schemas/Level"},"target":{"$ref":"#/components/schemas/Level"}},"example":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"ListSinkVersionsRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","afterId":"my-object-123","limit":100},"required":["branchId","sourceId","sinkId"]},"ListSinksRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","afterId":"my-object-123","limit":100},"required":["branchId","sourceId"]},"ListSourceVersionsRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","afterId":"my-object-123","limit":100},"required":["branchId","sourceId"]},"ListSourcesRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100}},"example":{"branchId":"default","afterId":"my-object-123","limit":100},"required":["branchId"]},"PaginatedResponse":{"type":"object","properties":{"afterId":{"type":"string","description":"Current offset.","example":"my-object-123"},"lastId":{"type":"string","description":"ID of the last record in the response.","example":"my-object-456"},"limit":{"type":"integer","description":"Current limit.","example":100,"format":"int64"},"totalCount":{"type":"integer","description":"Total count of all records.","example":1000,"format":"int64"}},"example":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"required":["afterId","limit","lastId","totalCount"]},"ServiceDetail":{"type":"object","properties":{"api":{"type":"string","description":"Name of the API","example":"stream"},"documentation":{"type":"string","description":"URL of the API documentation.","example":"https://stream.keboola.com/v1/documentation"}},"description":"Information about the service.","example":{"api":"stream","documentation":"https://stream.keboola.com/v1/documentation"},"required":["api","documentation"]},"SettingPatch":{"type":"object","properties":{"key":{"type":"string","description":"Key path.","example":"some.service.limit","minLength":1},"value":{"description":"A new key value. Use null to reset the value to the default value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","value":"1m20s"},"required":["key"]},"SettingResult":{"type":"object","properties":{"defaultValue":{"description":"Default value.","example":"30s"},"description":{"type":"string","description":"Key description.","example":"Minimal interval between uploads."},"key":{"type":"string","description":"Key path.","example":"some.service.limit"},"overwritten":{"type":"boolean","description":"True, if the default value is locally overwritten.","example":true},"protected":{"type":"boolean","description":"True, if only a super admin can modify the key.","example":false},"type":{"type":"string","description":"Value type.","example":"string","enum":["string","int","float","bool","[]string","[]int","[]float"]},"validation":{"type":"string","description":"Validation rules as a string definition.","example":"minDuration=15s"},"value":{"description":"Actual value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"},"required":["key","type","description","value","defaultValue","overwritten","protected"]},"SettingsPatch":{"type":"array","items":{"$ref":"#/components/schemas/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]},"SettingsResult":{"type":"object","properties":{"settings":{"type":"array","items":{"$ref":"#/components/schemas/SettingResult"},"example":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"Sink":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"A mapping from imported data to a destination table.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"SinkFile":{"type":"object","properties":{"closingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"openedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAfter":{"type":"string","description":"Next attempt time.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAttempt":{"type":"integer","description":"Number of failed attempts.","example":3,"format":"int64"},"retryReason":{"type":"string","description":"Reason of the last failed attempt.","example":"network problem"},"state":{"type":"string","example":"writing","enum":["writing","closing","importing","imported"]},"statistics":{"$ref":"#/components/schemas/SinkFileStatistics"}},"example":{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}},"required":["state","openedAt"]},"SinkFileStatistics":{"type":"object","properties":{"levels":{"$ref":"#/components/schemas/Levels"},"total":{"$ref":"#/components/schemas/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["total","levels"]},"SinkFiles":{"type":"array","items":{"$ref":"#/components/schemas/SinkFile"},"description":"List of recent sink files.","example":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"SinkSettingsPatch":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"$ref":"#/components/schemas/SettingsPatch"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]},"required":["branchId","sourceId","sinkId"]},"SinkStatisticsFilesResult":{"type":"object","properties":{"files":{"$ref":"#/components/schemas/SinkFiles"}},"example":{"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["files"]},"SinkStatisticsRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"failedFiles":{"type":"boolean","description":"Filter for not imported files. If set to true, only not imported files will be included.","default":false,"example":false},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","failedFiles":false},"required":["branchId","sourceId","sinkId"]},"SinkStatisticsTotalResult":{"type":"object","properties":{"levels":{"$ref":"#/components/schemas/Levels"},"total":{"$ref":"#/components/schemas/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["levels","total"]},"SinkVersionRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"versionNumber":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","versionNumber":3},"required":["branchId","sourceId","sinkId","versionNumber"]},"Sinks":{"type":"array","items":{"$ref":"#/components/schemas/Sink"},"description":"List of sinks, max 100 sinks per a source.","example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"SinksList":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"page":{"$ref":"#/components/schemas/PaginatedResponse"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinks":{"$ref":"#/components/schemas/Sinks"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"description":"List of sources, max 100 sinks per a source.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"required":["projectId","branchId","sourceId","page","sinks"]},"Source":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"http":{"$ref":"#/components/schemas/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}},"required":["projectId","branchId","sourceId","type","name","description","version","created"]},"SourceSettingsPatch":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"$ref":"#/components/schemas/SettingsPatch"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]},"required":["branchId","sourceId"]},"SourceVersionRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"versionNumber":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"example":{"branchId":"default","sourceId":"github-webhook-source","versionNumber":3},"required":["branchId","sourceId","versionNumber"]},"Sources":{"type":"array","items":{"$ref":"#/components/schemas/Source"},"description":"List of sources, max 100 sources per a branch.","example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"SourcesList":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"page":{"$ref":"#/components/schemas/PaginatedResponse"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sources":{"$ref":"#/components/schemas/Sources"}},"description":"List of sources, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github.","http":{"url":"https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb"},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"required":["projectId","branchId","page","sources"]},"TableColumn":{"type":"object","properties":{"defaultValue":{"type":"string","description":"Fallback value if path doesn't exist.","example":"1"},"name":{"type":"string","description":"Column name.","example":"id-col"},"path":{"type":"string","description":"Path to the value.","example":"foo.bar[0]"},"rawString":{"type":"boolean","description":"Set to true if path value should use raw string instead of json-encoded value.","example":true},"template":{"$ref":"#/components/schemas/TableColumnTemplate"},"type":{"type":"string","description":"Column mapping type. This represents a static mapping (e.g. `body` or `headers`), or a custom mapping using a template language (`template`).","example":"body","enum":["uuid","datetime","ip","body","headers","path","template"]}},"description":"An output mapping defined by a template.","example":{"type":"body","name":"id-col","path":"foo.bar[0]","defaultValue":"1","rawString":true,"template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}},"required":["type","name"]},"TableColumnTemplate":{"type":"object","properties":{"content":{"type":"string","example":"body.foo + \"-\" + body.bar","minLength":1,"maxLength":4096},"language":{"type":"string","example":"jsonnet","enum":["jsonnet"]}},"description":"Template column definition, for \"type\" = \"template\".","example":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"},"required":["language","content"]},"TableColumns":{"type":"array","items":{"$ref":"#/components/schemas/TableColumn"},"description":"List of export column mappings. An export may have a maximum of 100 columns.","example":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}],"minItems":1,"maxItems":100},"TableMapping":{"type":"object","properties":{"columns":{"$ref":"#/components/schemas/TableColumns"}},"description":"Table mapping definition.","example":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]},"required":["columns"]},"TableSink":{"type":"object","properties":{"mapping":{"$ref":"#/components/schemas/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkCreate":{"type":"object","properties":{"mapping":{"$ref":"#/components/schemas/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkUpdate":{"type":"object","properties":{"mapping":{"$ref":"#/components/schemas/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"Task":{"type":"object","properties":{"createdAt":{"type":"string","description":"Date and time of the task creation.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"duration":{"type":"integer","description":"Duration of the task in milliseconds.","example":123456789,"format":"int64"},"error":{"type":"string","example":"abc123"},"finishedAt":{"type":"string","description":"Date and time of the task end.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"isFinished":{"type":"boolean","description":"Shortcut for status != \"processing\".","example":false},"outputs":{"$ref":"#/components/schemas/TaskOutputs"},"result":{"type":"string","example":"abc123"},"status":{"type":"string","description":"Task status, one of: processing, success, error","example":"success","enum":["processing","success","error"]},"taskId":{"type":"string","description":"Unique ID of the task.","example":"task_1234"},"type":{"type":"string","description":"Task type.","example":"abc123"},"url":{"type":"string","description":"URL of the task.","example":"abc123"}},"description":"An asynchronous task.","example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"required":["taskId","type","url","status","isFinished","createdAt"]},"TaskOutputs":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"url":{"type":"string","description":"Absolute URL of the entity.","example":"abc123"}},"description":"Outputs generated by the task.","example":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"TestResult":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"tables":{"type":"array","items":{"$ref":"#/components/schemas/TestResultTable"},"description":"Table for each configured sink.","example":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]}},"description":"Result of the test endpoint.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","tables":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]},"required":["projectId","branchId","sourceId","tables"]},"TestResultColumn":{"type":"object","properties":{"name":{"type":"string","description":"Column name.","example":"id"},"value":{"type":"string","description":"Column value.","example":"12345"}},"description":"Generated table column value, part of the test result.","example":{"name":"id","value":"12345"},"required":["name","value"]},"TestResultRow":{"type":"object","properties":{"columns":{"type":"array","items":{"$ref":"#/components/schemas/TestResultColumn"},"description":"Generated columns.","example":[{"name":"id","value":"12345"}]}},"description":"Generated table row, part of the test result.","example":{"columns":[{"name":"id","value":"12345"}]},"required":["columns"]},"TestResultTable":{"type":"object","properties":{"rows":{"type":"array","items":{"$ref":"#/components/schemas/TestResultRow"},"description":"Generated rows.","example":[{"columns":[{"name":"id","value":"12345"}]}]},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"tableId":{"type":"string","example":"in.c-bucket.table"}},"description":"Generated table rows, part of the test result.","example":{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]},"required":["sinkId","tableId","rows"]},"TestSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source"},"required":["branchId","sourceId"]},"UpdateSinkRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSinkUpdate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["branchId","sourceId","sinkId"]},"UpdateSinkRequestBody":{"type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"table":{"$ref":"#/components/schemas/TableSinkUpdate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}},"UpdateSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["branchId","sourceId"]},"UpdateSourceRequestBody":{"type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"type":{"type":"string","example":"http","enum":["http"]}},"example":{"changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}},"UpdateSourceSettingsRequestBody":{"type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"type":"array","items":{"$ref":"#/components/schemas/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]}},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}},"Version":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of the modification.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"},"description":{"type":"string","description":"Description of the change.","example":"The reason for the last change was..."},"hash":{"type":"string","description":"Hash of the entity state.","example":"f43e93acd97eceb3"},"number":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"description":"Version of the entity.","example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["number","hash","at","by","description"]}},"securitySchemes":{"storage-api-token":{"type":"apiKey","description":"Storage Api Token Authentication.","name":"X-StorageApi-Token","in":"header"}}},"tags":[{"name":"stream","description":"A service for continuously importing data to the Keboola platform."}]} \ No newline at end of file +{"openapi":"3.0.3","info":{"title":"Stream Service","description":"A service for continuously importing data to the Keboola platform.","license":{"name":"Apache 2.0","url":"http://www.apache.org/licenses/LICENSE-2.0"},"version":"1.0"},"servers":[{"url":"https://stream.{stack}","variables":{"stack":{"enum":["keboola.com","eu-central-1.keboola.com","north-europe.azure.keboola.com","eu-west-1.aws.keboola.dev","east-us-2.azure.keboola-testing.com"],"default":"keboola.com"}}},{"url":"http://localhost:8001"}],"paths":{"/":{"get":{"tags":["documentation"],"summary":"Redirect to /v1","description":"Redirect to /v1.","operationId":"ApiRootIndex","responses":{"301":{"description":"Moved Permanently response."}}}},"/v1":{"get":{"tags":["documentation"],"summary":"List API name and link to documentation.","description":"List API name and link to documentation.","operationId":"ApiVersionIndex","responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ServiceDetail"},"example":{"api":"stream","documentation":"https://stream.keboola.com/v1/documentation"}}}}}}},"/v1/branches/{branchId}/aggregation/sources":{"get":{"tags":["internal"],"summary":"Aggregation endpoint for sources","description":"Details about sources for the UI.","operationId":"AggregationSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/AggregatedSourcesResult"},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sinks":[],"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources":{"get":{"tags":["configuration"],"summary":"List all sources","description":"List all sources in the branch.","operationId":"ListSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SourcesList"},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]}}}}},"security":[{"storage-api-token":[]}]},"post":{"tags":["configuration"],"summary":"Create source","description":"Create a new source in the branch.","operationId":"CreateSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSourceRequestBody"},"example":{"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"409":{"description":"stream.api.sourceAlreadyExists: Source already exists in the branch.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":409,"error":"stream.api.sourceAlreadyExists","message":"Source already exists in the branch."}}}},"422":{"description":"stream.api.resourceLimitReached: Resource limit reached.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":422,"error":"stream.api.resourceLimitReached","message":"Maximum number of sources per project is 100."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/deleted":{"get":{"tags":["configuration"],"summary":"List all deleted sources","description":"List all deleted sources in the branch.","operationId":"ListDeletedSources","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SourcesList"},"example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}":{"delete":{"tags":["configuration"],"summary":"Delete source","description":"Delete the source.","operationId":"DeleteSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"get":{"tags":["configuration"],"summary":"Get source","description":"Get the source definition.","operationId":"GetSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Source"},"examples":{"http_source":{"summary":"http_source","description":"HTTP source response shape.","value":{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}},"otlp_source":{"summary":"otlp_source","description":"OTLP source response shape.","value":{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update source","description":"Update the source.","operationId":"UpdateSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSourceRequestBody"},"example":{"changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/disable":{"put":{"tags":["configuration"],"summary":"Disable source","description":"Disables the source.","operationId":"DisableSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/enable":{"put":{"tags":["configuration"],"summary":"Enable source","description":"Enables the source.","operationId":"EnableSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/settings":{"get":{"tags":["configuration"],"summary":"Get source settings","description":"Get source settings.","operationId":"GetSourceSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SettingsResult"},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update source settings","description":"Update source settings.","operationId":"UpdateSourceSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSourceSettingsRequestBody"},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.forbidden: Modification of protected settings is forbidden.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.forbidden","message":"Cannot modify protected keys: \"storage.level.local.encoding.compression.gzip.blockSize\"."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks":{"get":{"tags":["configuration"],"summary":"List sinks","description":"List all sinks in the source.","operationId":"ListSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinksList"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]},"post":{"tags":["configuration"],"summary":"Create sink","description":"Create a new sink in the source.","operationId":"CreateSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSinkRequestBody"},"example":{"sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}},"409":{"description":"stream.api.sinkAlreadyExists: Sink already exists in the source.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":409,"error":"stream.api.sinkAlreadyExists","message":"Sink already exists in the source."}}}},"422":{"description":"stream.api.resourceLimitReached: Resource limit reached.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":422,"error":"stream.api.resourceLimitReached","message":"Maximum number of sources per project is 100."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/deleted":{"get":{"tags":["configuration"],"summary":"List deleted sinks","description":"List all deleted sinks in the source.","operationId":"ListDeletedSinks","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinksList"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}":{"delete":{"tags":["configuration"],"summary":"Delete sink","description":"Delete the sink.","operationId":"DeleteSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]},"get":{"tags":["configuration"],"summary":"Get sink","description":"Get the sink definition.","operationId":"GetSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Sink"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update sink","description":"Update the sink.","operationId":"UpdateSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSinkRequestBody"},"example":{"changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/disable":{"put":{"tags":["configuration"],"summary":"Disable sink","description":"Disables the sink.","operationId":"DisableSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/enable":{"put":{"tags":["configuration"],"summary":"Enable sink","description":"Enables the sink.","operationId":"EnableSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/settings":{"get":{"tags":["configuration"],"summary":"Get sink settings","description":"Get the sink settings.","operationId":"GetSinkSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SettingsResult"},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]},"patch":{"tags":["configuration"],"summary":"Update sink settings","description":"Update sink settings.","operationId":"UpdateSinkSettings","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateSourceSettingsRequestBody"},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}}}},"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.forbidden: Modification of protected settings is forbidden.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.forbidden","message":"Cannot modify protected keys: \"storage.level.local.encoding.compression.gzip.blockSize\"."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear sink statistics","description":"Clears all statistics of the sink.","operationId":"SinkStatisticsClear","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response."},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/files":{"get":{"tags":["configuration"],"summary":"Sink files statistics","description":"Get files statistics of the sink.","operationId":"SinkStatisticsFiles","parameters":[{"name":"failedFiles","in":"query","description":"Filter for not imported files. If set to true, only not imported files will be included.","allowEmptyValue":true,"schema":{"type":"boolean","description":"Filter for not imported files. If set to true, only not imported files will be included.","default":false,"example":false},"example":false},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinkStatisticsFilesResult"},"example":{"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/statistics/total":{"get":{"tags":["configuration"],"summary":"Sink statistics total","description":"Get total statistics of the sink.","operationId":"SinkStatisticsTotal","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SinkStatisticsTotalResult"},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete sink","description":"Undelete the sink.","operationId":"UndeleteSink","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions":{"get":{"tags":["configuration"],"summary":"List sink versions","description":"List all sink versions.","operationId":"ListSinkVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EntityVersions"},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}}}}},"404":{"description":"stream.api.sinkNotFound: Sink not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sinkNotFound","message":"Sink \"github-changed-files\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Sink version detail","description":"Sink version detail.","operationId":"SinkVersionDetail","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Version"},"example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/sinks/{sinkId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback sink version","description":"Rollback sink version.","operationId":"RollbackSinkVersion","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"sinkId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"example":"github-pr-table-sink"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/statistics/clear":{"delete":{"tags":["configuration"],"summary":"Clear source statistics","description":"Clears all statistics of the source.","operationId":"SourceStatisticsClear","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response."},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/test":{"post":{"tags":["test"],"summary":"Test source payload mapping","description":"Tests configured mapping of the source and its sinks against an example request body.\n\nFor HTTP sources the body is the raw payload that an HTTP client would send (treated as a single record).\n\nFor OTLP sources the body must be a single **flattened OTLP record** — a JSON object with the same shape that the source produces internally for each log record, metric data point, or span. Attributes, resource, and scope are nested objects (not dotted keys); reference them in column mappings as `Body('attributes')['user.id']`, `Body('resource')['service.name']`, `Body('scope')['name']`, etc. Example flattened log record:\n```json\n{\n \"timestamp\": \"2024-01-15T10:30:00Z\",\n \"observed_timestamp\": \"2024-01-15T10:30:00Z\",\n \"severity_number\": 9,\n \"severity_text\": \"INFO\",\n \"body\": \"User logged in\",\n \"flags\": 0,\n \"attributes\": {\"user.id\": \"user-123\"},\n \"resource\": {\"service.name\": \"auth-service\"},\n \"scope\": {\"name\": \"github.com/my/auth\", \"version\": \"1.2.3\"}\n}\n```\nDo not send a raw OTLP protobuf or the multi-record envelope produced by an OTel SDK — the test endpoint intentionally evaluates one already-flattened record so the response is deterministic. For OTLP sources, the `signal` query parameter selects which signal type the request simulates for sink routing (`logs` by default); sinks whose `allowedSignals` filter rejects that signal are skipped in the result.","operationId":"TestSource","parameters":[{"name":"signal","in":"query","description":"OTLP signal type to simulate for sink routing.","allowEmptyValue":true,"schema":{"type":"string","description":"OTLP signal type — one of logs, metrics, or traces.","example":"logs","enum":["logs","metrics","traces"]},"example":"logs"},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestResult"},"example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","tables":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}},"422":{"description":"stream.api.invalidColumnValue: Invalid data for sink.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":422,"error":"stream.api.invalidColumnValue","message":"Invalid value for column \"name\": path \"name\" not found in the body"}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/undelete":{"put":{"tags":["configuration"],"summary":"Undelete source","description":"Undelete the source.","operationId":"UndeleteSource","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions":{"get":{"tags":["configuration"],"summary":"List source versions","description":"List all source versions.","operationId":"ListSourceVersions","parameters":[{"name":"afterId","in":"query","description":"Request records after the ID.","allowEmptyValue":true,"schema":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"example":"my-object-123"},{"name":"limit","in":"query","description":"Maximum number of returned records.","allowEmptyValue":true,"schema":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"example":100},{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EntityVersions"},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}}}}},"404":{"description":"stream.api.sourceNotFound: Source not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.sourceNotFound","message":"Source \"github-pull-requests\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}":{"get":{"tags":["configuration"],"summary":"Source version detail","description":"Source version detail.","operationId":"SourceVersionDetail","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Version"},"example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/branches/{branchId}/sources/{sourceId}/versions/{versionNumber}/rollback":{"put":{"tags":["configuration"],"summary":"Rollback source version","description":"Rollback source version.","operationId":"RollbackSourceVersion","parameters":[{"name":"branchId","in":"path","required":true,"schema":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"example":"default"},{"name":"sourceId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"example":"github-webhook-source"},{"name":"versionNumber","in":"path","description":"Version number counted from 1.","required":true,"schema":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1},"example":3}],"responses":{"202":{"description":"Accepted response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.versionNotFound: Version not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.versionNotFound","message":"Version \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}},"/v1/documentation/openapi.json":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 JSON Specification","operationId":"OpenapiJson","responses":{"200":{"description":"File downloaded"}}}},"/v1/documentation/openapi.yaml":{"get":{"tags":["documentation"],"summary":"Swagger 2.0 YAML Specification","operationId":"OpenapiYaml","responses":{"200":{"description":"File downloaded"}}}},"/v1/documentation/openapi3.json":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 JSON Specification","operationId":"Openapi3Json","responses":{"200":{"description":"File downloaded"}}}},"/v1/documentation/openapi3.yaml":{"get":{"tags":["documentation"],"summary":"OpenAPI 3.0 YAML Specification","operationId":"Openapi3Yaml","responses":{"200":{"description":"File downloaded"}}}},"/v1/tasks/{taskId}":{"get":{"tags":["configuration"],"summary":"Get task","description":"Get details of a task.","operationId":"GetTask","parameters":[{"name":"taskId","in":"path","required":true,"schema":{"type":"string","description":"Unique ID of the task.","example":"task_1234"},"example":"task_1234"}],"responses":{"200":{"description":"OK response.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Task"},"example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}}}}},"404":{"description":"stream.api.taskNotFound: Task not found error.","content":{"application/json":{"schema":{"$ref":"#/components/schemas/GenericError"},"example":{"statusCode":404,"error":"stream.api.taskNotFound","message":"Task \"001\" not found."}}}}},"security":[{"storage-api-token":[]}]}}},"components":{"schemas":{"AggregatedSink":{"type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","description":"OTLP signal type — one of logs, metrics, or traces.","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"statistics":{"$ref":"#/components/schemas/AggregatedStatistics"},"table":{"$ref":"#/components/schemas/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"A mapping from imported data to a destination table.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"AggregatedSinks":{"type":"array","items":{"$ref":"#/components/schemas/AggregatedSink"},"example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]}}]},"AggregatedSource":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"http":{"$ref":"#/components/schemas/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"otlp":{"$ref":"#/components/schemas/OTLPSource"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinks":{"$ref":"#/components/schemas/AggregatedSinks"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},"required":["projectId","branchId","sourceId","type","name","description","version","created","sinks"]},"AggregatedSources":{"type":"array","items":{"$ref":"#/components/schemas/AggregatedSource"},"example":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sinks":[],"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]},"AggregatedSourcesRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100}},"example":{"branchId":"default","afterId":"my-object-123","limit":100},"required":["branchId"]},"AggregatedSourcesResult":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"page":{"$ref":"#/components/schemas/PaginatedResponse"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sources":{"$ref":"#/components/schemas/AggregatedSources"}},"description":"List of sources, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sinks":[],"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sinks":[],"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]},"required":["projectId","branchId","page","sources"]},"AggregatedStatistics":{"type":"object","properties":{"files":{"$ref":"#/components/schemas/SinkFiles"},"levels":{"$ref":"#/components/schemas/Levels"},"total":{"$ref":"#/components/schemas/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}},"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["total","levels","files"]},"By":{"type":"object","properties":{"tokenDesc":{"type":"string","description":"Description of the token.","example":"john.green@company.com"},"tokenId":{"type":"string","description":"ID of the token.","example":"896455"},"type":{"type":"string","description":"Date and time of deletion.","example":"user","enum":["system","user"]},"userId":{"type":"string","description":"ID of the user.","example":"578621"},"userName":{"type":"string","description":"Name of the user.","example":"John Green"}},"description":"Information about the operation actor.","example":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"required":["type"]},"CreateSinkRequest":{"type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","description":"OTLP signal type — one of logs, metrics, or traces.","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSinkCreate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["branchId","sourceId","type","name"]},"CreateSinkRequestBody":{"type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSinkCreate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["type","name"]},"CreateSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["branchId","type","name"]},"CreateSourceRequestBody":{"type":"object","properties":{"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Optional ID, if not filled in, it will be generated from name. Cannot be changed later.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]}},"example":{"sourceId":"github-webhook-source","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["type","name"]},"CreatedEntity":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"}},"description":"Information about the entity creation.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DeletedEntity":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of deletion.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"}},"description":"Information about the deleted entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["at","by"]},"DisabledEntity":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of disabling.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"},"reason":{"type":"string","description":"Why was the entity disabled?","example":"Disabled for recurring problems."}},"description":"Information about the disabled entity.","example":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."},"required":["at","by","reason"]},"EntityVersions":{"type":"object","properties":{"page":{"$ref":"#/components/schemas/PaginatedResponse"},"versions":{"type":"array","items":{"$ref":"#/components/schemas/Version"},"example":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}]}},"example":{"versions":[{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}}],"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"}},"required":["versions","page"]},"GenericError":{"type":"object","properties":{"error":{"type":"string","description":"Name of error.","example":"stream.internalError"},"message":{"type":"string","description":"Error message.","example":"Internal Error"},"statusCode":{"type":"integer","description":"HTTP status code.","example":500,"format":"int64"}},"description":"Generic error.","example":{"statusCode":500,"error":"stream.internalError","message":"Internal Error"},"required":["statusCode","error","message"]},"GetSinkRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"},"required":["branchId","sourceId","sinkId"]},"GetSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source"},"required":["branchId","sourceId"]},"GetTaskRequest":{"type":"object","properties":{"taskId":{"type":"string","description":"Unique ID of the task.","example":"task_1234"}},"example":{"taskId":"task_1234"},"required":["taskId"]},"HTTPSource":{"type":"object","properties":{"url":{"type":"string","description":"URL of the HTTP source. Contains secret used for authentication.","example":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"}},"description":"HTTP source details for \"type\" = \"http\".","example":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"required":["url"]},"Level":{"type":"object","properties":{"compressedSize":{"type":"integer","description":"Compressed size of data in bytes.","example":1,"format":"int64"},"firstRecordAt":{"type":"string","description":"Timestamp of the first received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"lastRecordAt":{"type":"string","description":"Timestamp of the last received record.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"recordsCount":{"type":"integer","example":1,"format":"int64"},"uncompressedSize":{"type":"integer","description":"Uncompressed size of data in bytes.","example":1,"format":"int64"}},"example":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"required":["recordsCount","compressedSize","uncompressedSize"]},"Levels":{"type":"object","properties":{"local":{"$ref":"#/components/schemas/Level"},"staging":{"$ref":"#/components/schemas/Level"},"target":{"$ref":"#/components/schemas/Level"}},"example":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"ListSinkVersionsRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","afterId":"my-object-123","limit":100},"required":["branchId","sourceId","sinkId"]},"ListSinksRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","afterId":"my-object-123","limit":100},"required":["branchId","sourceId"]},"ListSourceVersionsRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","afterId":"my-object-123","limit":100},"required":["branchId","sourceId"]},"ListSourcesRequest":{"type":"object","properties":{"afterId":{"type":"string","description":"Request records after the ID.","default":"","example":"my-object-123"},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"limit":{"type":"integer","description":"Maximum number of returned records.","default":100,"example":100,"format":"int64","minimum":1,"maximum":100}},"example":{"branchId":"default","afterId":"my-object-123","limit":100},"required":["branchId"]},"OTLPSource":{"type":"object","properties":{"baseUrl":{"type":"string","description":"Endpoint URL without the secret. Use this together with the `secret` field via the `Authorization: Bearer \u003csecret\u003e` header so the secret stays out of access/CDN/APM logs. The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.","example":"https://stream-in.keboola.com/otlp/123/my-source"},"secret":{"type":"string","description":"48-character secret authenticating writes to this source. Send it as `Authorization: Bearer \u003csecret\u003e` to the `baseUrl`.","example":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"url":{"type":"string","description":"Endpoint URL with the secret embedded as the last path segment. Convenient for SDKs that authenticate by URL only. The OpenTelemetry SDK automatically appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — do not append a signal path yourself. Most SDK exporters reject or silently strip the suffix.","example":"https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"}},"description":"OTLP/HTTP source details for \"type\" = \"otlp\".","example":{"url":"https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","baseUrl":"https://stream-in.keboola.com/otlp/123/my-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"required":["url","baseUrl","secret"]},"PaginatedResponse":{"type":"object","properties":{"afterId":{"type":"string","description":"Current offset.","example":"my-object-123"},"lastId":{"type":"string","description":"ID of the last record in the response.","example":"my-object-456"},"limit":{"type":"integer","description":"Current limit.","example":100,"format":"int64"},"totalCount":{"type":"integer","description":"Total count of all records.","example":1000,"format":"int64"}},"example":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"required":["afterId","limit","lastId","totalCount"]},"ServiceDetail":{"type":"object","properties":{"api":{"type":"string","description":"Name of the API","example":"stream"},"documentation":{"type":"string","description":"URL of the API documentation.","example":"https://stream.keboola.com/v1/documentation"}},"description":"Information about the service.","example":{"api":"stream","documentation":"https://stream.keboola.com/v1/documentation"},"required":["api","documentation"]},"SettingPatch":{"type":"object","properties":{"key":{"type":"string","description":"Key path.","example":"some.service.limit","minLength":1},"value":{"description":"A new key value. Use null to reset the value to the default value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","value":"1m20s"},"required":["key"]},"SettingResult":{"type":"object","properties":{"defaultValue":{"description":"Default value.","example":"30s"},"description":{"type":"string","description":"Key description.","example":"Minimal interval between uploads."},"key":{"type":"string","description":"Key path.","example":"some.service.limit"},"overwritten":{"type":"boolean","description":"True, if the default value is locally overwritten.","example":true},"protected":{"type":"boolean","description":"True, if only a super admin can modify the key.","example":false},"type":{"type":"string","description":"Value type.","example":"string","enum":["string","int","float","bool","[]string","[]int","[]float"]},"validation":{"type":"string","description":"Validation rules as a string definition.","example":"minDuration=15s"},"value":{"description":"Actual value.","example":"1m20s"}},"description":"One setting key-value pair.","example":{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"},"required":["key","type","description","value","defaultValue","overwritten","protected"]},"SettingsPatch":{"type":"array","items":{"$ref":"#/components/schemas/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]},"SettingsResult":{"type":"object","properties":{"settings":{"type":"array","items":{"$ref":"#/components/schemas/SettingResult"},"example":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"example":{"settings":[{"key":"some.service.limit","type":"string","description":"Minimal interval between uploads.","value":"1m20s","defaultValue":"30s","overwritten":true,"protected":false,"validation":"minDuration=15s"}]}},"Sink":{"type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","description":"OTLP signal type — one of logs, metrics, or traces.","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSink"},"type":{"type":"string","example":"table","enum":["table"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"A mapping from imported data to a destination table.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}},"required":["projectId","branchId","sourceId","sinkId","type","name","description","version","created"]},"SinkFile":{"type":"object","properties":{"closingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"importingAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"openedAt":{"type":"string","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAfter":{"type":"string","description":"Next attempt time.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"retryAttempt":{"type":"integer","description":"Number of failed attempts.","example":3,"format":"int64"},"retryReason":{"type":"string","description":"Reason of the last failed attempt.","example":"network problem"},"state":{"type":"string","example":"writing","enum":["writing","closing","importing","imported"]},"statistics":{"$ref":"#/components/schemas/SinkFileStatistics"}},"example":{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}},"required":["state","openedAt"]},"SinkFileStatistics":{"type":"object","properties":{"levels":{"$ref":"#/components/schemas/Levels"},"total":{"$ref":"#/components/schemas/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["total","levels"]},"SinkFiles":{"type":"array","items":{"$ref":"#/components/schemas/SinkFile"},"description":"List of recent sink files.","example":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"SinkSettingsPatch":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"$ref":"#/components/schemas/SettingsPatch"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]},"required":["branchId","sourceId","sinkId"]},"SinkStatisticsFilesResult":{"type":"object","properties":{"files":{"$ref":"#/components/schemas/SinkFiles"}},"example":{"files":[{"state":"writing","openedAt":"2022-04-28T14:20:04.000Z","closingAt":"2022-04-28T14:20:04.000Z","importingAt":"2022-04-28T14:20:04.000Z","importedAt":"2022-04-28T14:20:04.000Z","retryAttempt":3,"retryReason":"network problem","retryAfter":"2022-04-28T14:20:04.000Z","statistics":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}}}]},"required":["files"]},"SinkStatisticsRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"failedFiles":{"type":"boolean","description":"Filter for not imported files. If set to true, only not imported files will be included.","default":false,"example":false},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","failedFiles":false},"required":["branchId","sourceId","sinkId"]},"SinkStatisticsTotalResult":{"type":"object","properties":{"levels":{"$ref":"#/components/schemas/Levels"},"total":{"$ref":"#/components/schemas/Level"}},"example":{"total":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"levels":{"local":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"staging":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1},"target":{"firstRecordAt":"2022-04-28T14:20:04.000Z","lastRecordAt":"2022-04-28T14:20:04.000Z","recordsCount":1,"compressedSize":1,"uncompressedSize":1}}},"required":["levels","total"]},"SinkVersionRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"versionNumber":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","versionNumber":3},"required":["branchId","sourceId","sinkId","versionNumber"]},"Sinks":{"type":"array","items":{"$ref":"#/components/schemas/Sink"},"description":"List of sinks, max 100 sinks per a source.","example":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"SinksList":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"page":{"$ref":"#/components/schemas/PaginatedResponse"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinks":{"$ref":"#/components/schemas/Sinks"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"description":"List of sources, max 100 sinks per a source.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sinks":[{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"version":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"created":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"deleted":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"disabled":{"at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"},"reason":"Disabled for recurring problems."}}]},"required":["projectId","branchId","sourceId","page","sinks"]},"Source":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"created":{"$ref":"#/components/schemas/CreatedEntity"},"deleted":{"$ref":"#/components/schemas/DeletedEntity"},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"disabled":{"$ref":"#/components/schemas/DisabledEntity"},"http":{"$ref":"#/components/schemas/HTTPSource"},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"otlp":{"$ref":"#/components/schemas/OTLPSource"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]},"version":{"$ref":"#/components/schemas/Version"}},"description":"Source of data for further processing, start of the stream, max 100 sources per a branch.","example":{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},"required":["projectId","branchId","sourceId","type","name","description","version","created"]},"SourceSettingsPatch":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"$ref":"#/components/schemas/SettingsPatch"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]},"required":["branchId","sourceId"]},"SourceVersionRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"versionNumber":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"example":{"branchId":"default","sourceId":"github-webhook-source","versionNumber":3},"required":["branchId","sourceId","versionNumber"]},"Sources":{"type":"array","items":{"$ref":"#/components/schemas/Source"},"description":"List of sources, max 100 sources per a branch.","example":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]},"SourcesList":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"page":{"$ref":"#/components/schemas/PaginatedResponse"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sources":{"$ref":"#/components/schemas/Sources"}},"description":"List of sources, max 100 sources per a branch.","example":{"projectId":123,"branchId":345,"page":{"limit":100,"totalCount":1000,"afterId":"my-object-123","lastId":"my-object-456"},"sources":[{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","http":{"url":"https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"name":"My HTTP Source","projectId":1234,"sourceId":"my-http-source","type":"http","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}},{"branchId":5678,"created":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"}},"description":"","name":"My OTLP Source","otlp":{"baseUrl":"https://stream-in.keboola.com/otlp/1234/my-otlp-source","secret":"EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX","url":"https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX"},"projectId":1234,"sourceId":"my-otlp-source","type":"otlp","version":{"at":"2024-01-15T10:00:00.000Z","by":{"tokenDesc":"john.green@company.com","tokenId":"896455","type":"user","userId":"578621","userName":"John Green"},"description":"New source.","hash":"f43e93acd97eceb3","number":1}}]},"required":["projectId","branchId","page","sources"]},"TableColumn":{"type":"object","properties":{"defaultValue":{"type":"string","description":"Fallback value if path doesn't exist.","example":"1"},"name":{"type":"string","description":"Column name.","example":"id-col"},"path":{"type":"string","description":"Path to the value.","example":"foo.bar[0]"},"rawString":{"type":"boolean","description":"Set to true if path value should use raw string instead of json-encoded value.","example":true},"template":{"$ref":"#/components/schemas/TableColumnTemplate"},"type":{"type":"string","description":"Column mapping type. This represents a static mapping (e.g. `body` or `headers`), or a custom mapping using a template language (`template`).","example":"body","enum":["uuid","datetime","ip","body","headers","path","template"]}},"description":"An output mapping defined by a template.","example":{"type":"body","name":"id-col","path":"foo.bar[0]","defaultValue":"1","rawString":true,"template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}},"required":["type","name"]},"TableColumnTemplate":{"type":"object","properties":{"content":{"type":"string","example":"body.foo + \"-\" + body.bar","minLength":1,"maxLength":4096},"language":{"type":"string","example":"jsonnet","enum":["jsonnet"]}},"description":"Template column definition, for \"type\" = \"template\".","example":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"},"required":["language","content"]},"TableColumns":{"type":"array","items":{"$ref":"#/components/schemas/TableColumn"},"description":"List of export column mappings. An export may have a maximum of 100 columns.","example":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}],"minItems":1,"maxItems":100},"TableMapping":{"type":"object","properties":{"columns":{"$ref":"#/components/schemas/TableColumns"}},"description":"Table mapping definition.","example":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]},"required":["columns"]},"TableSink":{"type":"object","properties":{"mapping":{"$ref":"#/components/schemas/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkCreate":{"type":"object","properties":{"mapping":{"$ref":"#/components/schemas/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}},"required":["type","tableId","mapping"]},"TableSinkUpdate":{"type":"object","properties":{"mapping":{"$ref":"#/components/schemas/TableMapping"},"tableId":{"type":"string","example":"in.c-bucket.table"},"type":{"type":"string","example":"keboola","enum":["keboola"]}},"description":"Table sink configuration for \"type\" = \"table\".","example":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"Task":{"type":"object","properties":{"createdAt":{"type":"string","description":"Date and time of the task creation.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"duration":{"type":"integer","description":"Duration of the task in milliseconds.","example":123456789,"format":"int64"},"error":{"type":"string","example":"abc123"},"finishedAt":{"type":"string","description":"Date and time of the task end.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"isFinished":{"type":"boolean","description":"Shortcut for status != \"processing\".","example":false},"outputs":{"$ref":"#/components/schemas/TaskOutputs"},"result":{"type":"string","example":"abc123"},"status":{"type":"string","description":"Task status, one of: processing, success, error","example":"success","enum":["processing","success","error"]},"taskId":{"type":"string","description":"Unique ID of the task.","example":"task_1234"},"type":{"type":"string","description":"Task type.","example":"abc123"},"url":{"type":"string","description":"URL of the task.","example":"abc123"}},"description":"An asynchronous task.","example":{"taskId":"task_1234","type":"abc123","url":"abc123","status":"success","isFinished":false,"createdAt":"2022-04-28T14:20:04.000Z","finishedAt":"2022-04-28T14:20:04.000Z","duration":123456789,"result":"abc123","error":"abc123","outputs":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"required":["taskId","type","url","status","isFinished","createdAt"]},"TaskOutputs":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"url":{"type":"string","description":"Absolute URL of the entity.","example":"abc123"}},"description":"Outputs generated by the task.","example":{"url":"abc123","projectId":123,"branchId":345,"sourceId":"github-webhook-source","sinkId":"github-pr-table-sink"}},"TestResult":{"type":"object","properties":{"branchId":{"type":"integer","description":"ID of the branch.","example":345,"format":"int64"},"projectId":{"type":"integer","description":"ID of the project.","example":123,"format":"int64"},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"tables":{"type":"array","items":{"$ref":"#/components/schemas/TestResultTable"},"description":"Table for each configured sink.","example":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]}},"description":"Result of the test endpoint.","example":{"projectId":123,"branchId":345,"sourceId":"github-webhook-source","tables":[{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]}]},"required":["projectId","branchId","sourceId","tables"]},"TestResultColumn":{"type":"object","properties":{"name":{"type":"string","description":"Column name.","example":"id"},"value":{"type":"string","description":"Column value.","example":"12345"}},"description":"Generated table column value, part of the test result.","example":{"name":"id","value":"12345"},"required":["name","value"]},"TestResultRow":{"type":"object","properties":{"columns":{"type":"array","items":{"$ref":"#/components/schemas/TestResultColumn"},"description":"Generated columns.","example":[{"name":"id","value":"12345"}]}},"description":"Generated table row, part of the test result.","example":{"columns":[{"name":"id","value":"12345"}]},"required":["columns"]},"TestResultTable":{"type":"object","properties":{"rows":{"type":"array","items":{"$ref":"#/components/schemas/TestResultRow"},"description":"Generated rows.","example":[{"columns":[{"name":"id","value":"12345"}]}]},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"tableId":{"type":"string","example":"in.c-bucket.table"}},"description":"Generated table rows, part of the test result.","example":{"sinkId":"github-pr-table-sink","tableId":"in.c-bucket.table","rows":[{"columns":[{"name":"id","value":"12345"}]}]},"required":["sinkId","tableId","rows"]},"TestSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"signal":{"type":"string","description":"OTLP signal type — one of logs, metrics, or traces.","example":"logs","enum":["logs","metrics","traces"]},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48}},"example":{"branchId":"default","sourceId":"github-webhook-source","signal":"logs"},"required":["branchId","sourceId"]},"UpdateSinkRequest":{"type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","description":"OTLP signal type — one of logs, metrics, or traces.","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"sinkId":{"type":"string","description":"Unique ID of the sink.","example":"github-pr-table-sink","minLength":1,"maxLength":48},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"table":{"$ref":"#/components/schemas/TableSinkUpdate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","sinkId":"github-pr-table-sink","changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}},"required":["branchId","sourceId","sinkId"]},"UpdateSinkRequestBody":{"type":"object","properties":{"allowedSignals":{"type":"array","items":{"type":"string","example":"logs","enum":["logs","metrics","traces"]},"description":"Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field.","example":["logs"]},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The sink stores records to a table.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the sink.","example":"Raw Data Sink","minLength":1,"maxLength":40},"table":{"$ref":"#/components/schemas/TableSinkUpdate"},"type":{"type":"string","example":"table","enum":["table"]}},"example":{"changeDescription":"Renamed.","type":"table","name":"Raw Data Sink","description":"The sink stores records to a table.","allowedSignals":["logs"],"table":{"type":"keboola","tableId":"in.c-bucket.table","mapping":{"columns":[{"type":"uuid","name":"id-col"},{"type":"datetime","name":"datetime-col"},{"type":"ip","name":"ip-col"},{"type":"headers","name":"headers-col"},{"type":"body","name":"body-col"},{"type":"path","name":"path-col","path":"foo.bar[0]","rawString":true,"defaultValue":""},{"type":"template","name":"template-col","template":{"language":"jsonnet","content":"body.foo + \"-\" + body.bar"}}]}}}},"UpdateSourceRequest":{"type":"object","properties":{"branchId":{"type":"string","description":"ID of the branch or \"default\".","example":"default"},"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"sourceId":{"type":"string","description":"Unique ID of the source.","example":"github-webhook-source","minLength":1,"maxLength":48},"type":{"type":"string","example":"http","enum":["http","otlp"]}},"example":{"branchId":"default","sourceId":"github-webhook-source","changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."},"required":["branchId","sourceId"]},"UpdateSourceRequestBody":{"type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Renamed."},"description":{"type":"string","description":"Description of the source.","example":"The source receives events from Github.","maxLength":4096},"name":{"type":"string","description":"Human readable name of the source.","example":"Github Webhook Source","minLength":1,"maxLength":40},"type":{"type":"string","example":"http","enum":["http","otlp"]}},"example":{"changeDescription":"Renamed.","type":"http","name":"Github Webhook Source","description":"The source receives events from Github."}},"UpdateSourceSettingsRequestBody":{"type":"object","properties":{"changeDescription":{"type":"string","description":"Description of the modification, description of the version.","example":"Updated settings."},"settings":{"type":"array","items":{"$ref":"#/components/schemas/SettingPatch"},"example":[{"key":"some.service.limit","value":"1m20s"}]}},"example":{"changeDescription":"Updated settings.","settings":[{"key":"some.service.limit","value":"1m20s"}]}},"Version":{"type":"object","properties":{"at":{"type":"string","description":"Date and time of the modification.","example":"2022-04-28T14:20:04.000Z","format":"date-time"},"by":{"$ref":"#/components/schemas/By"},"description":{"type":"string","description":"Description of the change.","example":"The reason for the last change was..."},"hash":{"type":"string","description":"Hash of the entity state.","example":"f43e93acd97eceb3"},"number":{"type":"integer","description":"Version number counted from 1.","example":3,"format":"int64","minimum":1}},"description":"Version of the entity.","example":{"number":3,"hash":"f43e93acd97eceb3","description":"The reason for the last change was...","at":"2022-04-28T14:20:04.000Z","by":{"type":"user","tokenId":"896455","tokenDesc":"john.green@company.com","userId":"578621","userName":"John Green"}},"required":["number","hash","at","by","description"]}},"securitySchemes":{"storage-api-token":{"type":"apiKey","description":"Storage Api Token Authentication.","name":"X-StorageApi-Token","in":"header"}}},"tags":[{"name":"stream","description":"A service for continuously importing data to the Keboola platform."}]} \ No newline at end of file diff --git a/internal/pkg/service/stream/api/openapi/openapi3.yaml b/internal/pkg/service/stream/api/openapi/openapi3.yaml index 3819327e5e..75fd080dcf 100644 --- a/internal/pkg/service/stream/api/openapi/openapi3.yaml +++ b/internal/pkg/service/stream/api/openapi/openapi3.yaml @@ -101,180 +101,64 @@ paths: afterId: my-object-123 lastId: my-object-456 sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sinks: [] + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 + description: New source. + hash: f43e93acd97eceb3 + number: 1 security: - storage-api-token: [] /v1/branches/{branchId}/sources: @@ -332,50 +216,62 @@ paths: afterId: my-object-123 lastId: my-object-456 sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 security: - storage-api-token: [] post: @@ -544,51 +440,71 @@ paths: application/json: schema: $ref: '#/components/schemas/Source' - example: - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. + examples: + http_source: + summary: http_source + description: HTTP source response shape. + value: + branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 + otlp_source: + summary: otlp_source + description: OTLP source response shape. + value: + branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 "404": description: 'stream.api.sourceNotFound: Source not found error.' content: @@ -997,6 +913,8 @@ paths: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -1106,6 +1024,8 @@ paths: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -1309,6 +1229,8 @@ paths: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -1428,6 +1350,8 @@ paths: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -2413,6 +2337,8 @@ paths: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -2532,9 +2458,41 @@ paths: tags: - test summary: Test source payload mapping - description: Tests configured mapping of the source and its sinks. + description: |- + Tests configured mapping of the source and its sinks against an example request body. + + For HTTP sources the body is the raw payload that an HTTP client would send (treated as a single record). + + For OTLP sources the body must be a single **flattened OTLP record** — a JSON object with the same shape that the source produces internally for each log record, metric data point, or span. Attributes, resource, and scope are nested objects (not dotted keys); reference them in column mappings as `Body('attributes')['user.id']`, `Body('resource')['service.name']`, `Body('scope')['name']`, etc. Example flattened log record: + ```json + { + "timestamp": "2024-01-15T10:30:00Z", + "observed_timestamp": "2024-01-15T10:30:00Z", + "severity_number": 9, + "severity_text": "INFO", + "body": "User logged in", + "flags": 0, + "attributes": {"user.id": "user-123"}, + "resource": {"service.name": "auth-service"}, + "scope": {"name": "github.com/my/auth", "version": "1.2.3"} + } + ``` + Do not send a raw OTLP protobuf or the multi-record envelope produced by an OTel SDK — the test endpoint intentionally evaluates one already-flattened record so the response is deterministic. For OTLP sources, the `signal` query parameter selects which signal type the request simulates for sink routing (`logs` by default); sinks whose `allowedSignals` filter rejects that signal are skipped in the result. operationId: TestSource parameters: + - name: signal + in: query + description: OTLP signal type to simulate for sink routing. + allowEmptyValue: true + schema: + type: string + description: OTLP signal type — one of logs, metrics, or traces. + example: logs + enum: + - logs + - metrics + - traces + example: logs - name: branchId in: path required: true @@ -2935,50 +2893,62 @@ paths: afterId: my-object-123 lastId: my-object-456 sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 security: - storage-api-token: [] /v1/documentation/openapi.json: @@ -3074,6 +3044,19 @@ components: AggregatedSink: type: object properties: + allowedSignals: + type: array + items: + type: string + description: OTLP signal type — one of logs, metrics, or traces. + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs branchId: type: integer description: ID of the branch. @@ -3133,6 +3116,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -3277,6 +3262,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -3426,6 +3413,8 @@ components: example: Github Webhook Source minLength: 1 maxLength: 40 + otlp: + $ref: '#/components/schemas/OTLPSource' projectId: type: integer description: ID of the project. @@ -3444,184 +3433,41 @@ components: example: http enum: - http + - otlp version: $ref: '#/components/schemas/Version' description: Source of data for further processing, start of the stream, max 100 sources per a branch. example: - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -3637,180 +3483,64 @@ components: items: $ref: '#/components/schemas/AggregatedSource' example: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green + - branchId: 5678 created: - at: "2022-04-28T14:20:04.000Z" + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: type: user - tokenId: "896455" - tokenDesc: john.green@company.com userId: "578621" userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sinks: [] + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" + by: + tokenDesc: john.green@company.com + tokenId: "896455" + type: user + userId: "578621" + userName: John Green + description: New source. + hash: f43e93acd97eceb3 + number: 1 AggregatedSourcesRequest: type: object properties: @@ -3864,180 +3594,64 @@ components: afterId: my-object-123 lastId: my-object-456 sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sinks: [] + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sinks: [] + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. - sinks: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - sinkId: github-pr-table-sink - type: table - name: Raw Data Sink - description: The sink stores records to a table. - table: - type: keboola - tableId: in.c-bucket.table - mapping: - columns: - - type: uuid - name: id-col - - type: datetime - name: datetime-col - - type: ip - name: ip-col - - type: headers - name: headers-col - - type: body - name: body-col - - type: path - name: path-col - path: foo.bar[0] - rawString: true - defaultValue: "" - - type: template - name: template-col - template: - language: jsonnet - content: body.foo + "-" + body.bar - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - reason: Disabled for recurring problems. - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - files: - - state: writing - openedAt: "2022-04-28T14:20:04.000Z" - closingAt: "2022-04-28T14:20:04.000Z" - importingAt: "2022-04-28T14:20:04.000Z" - importedAt: "2022-04-28T14:20:04.000Z" - retryAttempt: 3 - retryReason: network problem - retryAfter: "2022-04-28T14:20:04.000Z" - statistics: - total: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - levels: - local: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - staging: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 - target: - firstRecordAt: "2022-04-28T14:20:04.000Z" - lastRecordAt: "2022-04-28T14:20:04.000Z" - recordsCount: 1 - compressedSize: 1 - uncompressedSize: 1 + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -4155,6 +3769,19 @@ components: CreateSinkRequest: type: object properties: + allowedSignals: + type: array + items: + type: string + description: OTLP signal type — one of logs, metrics, or traces. + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs branchId: type: string description: ID of the branch or "default". @@ -4196,6 +3823,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -4229,6 +3858,18 @@ components: CreateSinkRequestBody: type: object properties: + allowedSignals: + type: array + items: + type: string + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs description: type: string description: Description of the source. @@ -4258,6 +3899,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -4315,6 +3958,7 @@ components: example: http enum: - http + - otlp example: branchId: default sourceId: github-webhook-source @@ -4350,6 +3994,7 @@ components: example: http enum: - http + - otlp example: sourceId: github-webhook-source type: http @@ -4558,10 +4203,10 @@ components: url: type: string description: URL of the HTTP source. Contains secret used for authentication. - example: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb + example: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX description: HTTP source details for "type" = "http". example: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX required: - url Level: @@ -4765,6 +4410,30 @@ components: limit: 100 required: - branchId + OTLPSource: + type: object + properties: + baseUrl: + type: string + description: 'Endpoint URL without the secret. Use this together with the `secret` field via the `Authorization: Bearer ` header so the secret stays out of access/CDN/APM logs. The OpenTelemetry SDK appends /v1/logs, /v1/metrics, or /v1/traces automatically.' + example: https://stream-in.keboola.com/otlp/123/my-source + secret: + type: string + description: '48-character secret authenticating writes to this source. Send it as `Authorization: Bearer ` to the `baseUrl`.' + example: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: + type: string + description: Endpoint URL with the secret embedded as the last path segment. Convenient for SDKs that authenticate by URL only. The OpenTelemetry SDK automatically appends /v1/logs, /v1/metrics, or /v1/traces based on the signal type — do not append a signal path yourself. Most SDK exporters reject or silently strip the suffix. + example: https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + description: OTLP/HTTP source details for "type" = "otlp". + example: + url: https://stream-in.keboola.com/otlp/123/my-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + baseUrl: https://stream-in.keboola.com/otlp/123/my-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + required: + - url + - baseUrl + - secret PaginatedResponse: type: object properties: @@ -4926,6 +4595,19 @@ components: Sink: type: object properties: + allowedSignals: + type: array + items: + type: string + description: OTLP signal type — one of logs, metrics, or traces. + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs branchId: type: integer description: ID of the branch. @@ -4983,6 +4665,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -5408,6 +5092,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -5510,6 +5196,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -5604,6 +5292,8 @@ components: example: Github Webhook Source minLength: 1 maxLength: 40 + otlp: + $ref: '#/components/schemas/OTLPSource' projectId: type: integer description: ID of the project. @@ -5620,54 +5310,40 @@ components: example: http enum: - http + - otlp version: $ref: '#/components/schemas/Version' description: Source of data for further processing, start of the stream, max 100 sources per a branch. example: - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" - by: - type: user - tokenId: "896455" - tokenDesc: john.green@company.com - userId: "578621" - userName: John Green + branchId: 5678 created: - at: "2022-04-28T14:20:04.000Z" + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com - userId: "578621" - userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" - by: - type: user tokenId: "896455" - tokenDesc: john.green@company.com + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -5739,50 +5415,62 @@ components: $ref: '#/components/schemas/Source' description: List of sources, max 100 sources per a branch. example: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 SourcesList: type: object properties: @@ -5810,50 +5498,62 @@ components: afterId: my-object-123 lastId: my-object-456 sources: - - projectId: 123 - branchId: 345 - sourceId: github-webhook-source - type: http - name: Github Webhook Source - description: The source receives events from Github. - http: - url: https://stream-in.keboola.com/G0lpTbz0vhakDicfoDQQ3BCzGYdW3qewd1D3eUbqETygHKGb - version: - number: 3 - hash: f43e93acd97eceb3 - description: The reason for the last change was... - at: "2022-04-28T14:20:04.000Z" + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - created: - at: "2022-04-28T14:20:04.000Z" + description: "" + http: + url: https://stream-in.keboola.com/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + name: My HTTP Source + projectId: 1234 + sourceId: my-http-source + type: http + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - deleted: - at: "2022-04-28T14:20:04.000Z" + description: New source. + hash: f43e93acd97eceb3 + number: 1 + - branchId: 5678 + created: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - disabled: - at: "2022-04-28T14:20:04.000Z" + description: "" + name: My OTLP Source + otlp: + baseUrl: https://stream-in.keboola.com/otlp/1234/my-otlp-source + secret: EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + url: https://stream-in.keboola.com/otlp/1234/my-otlp-source/EXAMPLE-SECRET-PLACEHOLDER-XXXXXXXXXXXXXXXXXXXXX + projectId: 1234 + sourceId: my-otlp-source + type: otlp + version: + at: "2024-01-15T10:00:00.000Z" by: - type: user - tokenId: "896455" tokenDesc: john.green@company.com + tokenId: "896455" + type: user userId: "578621" userName: John Green - reason: Disabled for recurring problems. + description: New source. + hash: f43e93acd97eceb3 + number: 1 required: - projectId - branchId @@ -6343,6 +6043,14 @@ components: type: string description: ID of the branch or "default". example: default + signal: + type: string + description: OTLP signal type — one of logs, metrics, or traces. + example: logs + enum: + - logs + - metrics + - traces sourceId: type: string description: Unique ID of the source. @@ -6352,12 +6060,26 @@ components: example: branchId: default sourceId: github-webhook-source + signal: logs required: - branchId - sourceId UpdateSinkRequest: type: object properties: + allowedSignals: + type: array + items: + type: string + description: OTLP signal type — one of logs, metrics, or traces. + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs branchId: type: string description: ID of the branch or "default". @@ -6404,6 +6126,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -6436,6 +6160,18 @@ components: UpdateSinkRequestBody: type: object properties: + allowedSignals: + type: array + items: + type: string + example: logs + enum: + - logs + - metrics + - traces + description: Restricts the sink to specific OTLP signal types. Empty (default) accepts all signals. Only relevant for OTLP sources; HTTP sources ignore this field. + example: + - logs changeDescription: type: string description: Description of the modification, description of the version. @@ -6463,6 +6199,8 @@ components: type: table name: Raw Data Sink description: The sink stores records to a table. + allowedSignals: + - logs table: type: keboola tableId: in.c-bucket.table @@ -6521,6 +6259,7 @@ components: example: http enum: - http + - otlp example: branchId: default sourceId: github-webhook-source @@ -6554,6 +6293,7 @@ components: example: http enum: - http + - otlp example: changeDescription: Renamed. type: http diff --git a/internal/pkg/service/stream/api/service/source.go b/internal/pkg/service/stream/api/service/source.go index 9ea3be6316..6ad459ac5b 100644 --- a/internal/pkg/service/stream/api/service/source.go +++ b/internal/pkg/service/stream/api/service/source.go @@ -303,8 +303,9 @@ func (s *service) UpdateSourceSettings(ctx context.Context, d dependencies.Sourc return s.mapper.NewTaskResponse(t) } -func (s *service) TestSource(ctx context.Context, d dependencies.SourceRequestScope, _ *api.TestSourcePayload, _ io.ReadCloser) (res *api.TestResult, err error) { - if err := s.sourceMustExist(ctx, d.SourceKey()); err != nil { +func (s *service) TestSource(ctx context.Context, d dependencies.SourceRequestScope, payload *api.TestSourcePayload, _ io.ReadCloser) (res *api.TestResult, err error) { + source, err := s.definition.Source().Get(d.SourceKey()).Do(ctx).ResultOrErr() + if err != nil { return nil, err } @@ -317,11 +318,46 @@ func (s *service) TestSource(ctx context.Context, d dependencies.SourceRequestSc req := d.Request() req.Header.Del("x-storageapi-token") - recordCtx := recordctx.FromHTTP(d.Clock().Now(), req) + var recordCtx recordctx.Context + if source.Type == definition.SourceTypeOTLP { + // For OTLP sources the test body is interpreted as an already-flattened + // OTLP record (the same structure that FlattenLogs/Metrics/Traces produces). + // This lets API callers test their jsonnet column expressions against a + // realistic flat payload without having to send a real protobuf batch. + // The signal selector is validated by the Goa enum; default to "logs" + // when the query parameter is omitted. + signal := "logs" + if payload != nil && payload.Signal != nil { + signal = string(*payload.Signal) + } + recordCtx, err = recordctx.FromOTLPTestRequest(ctx, d.Clock().Now(), req, signal) + if err != nil { + return nil, svcerrors.NewBadRequestError(err) + } + // Sinks whose AllowedSignals filter rejects this signal would be skipped + // during real ingestion; rendering them here would surface mapping errors + // from sinks that never see this record at runtime. + sinks = filterSinksBySignal(sinks, recordCtx.Signal()) + } else { + recordCtx = recordctx.FromHTTP(d.Clock().Now(), req) + } return s.mapper.NewTestResultResponse(d.SourceKey(), sinks, recordCtx) } +// filterSinksBySignal keeps only sinks whose AcceptsSignal returns true. +// Delegates to definition.SignalAccepted so the /test endpoint stays in lock +// step with the runtime router's per-signal dispatch decision. +func filterSinksBySignal(sinks []definition.Sink, signal string) []definition.Sink { + out := make([]definition.Sink, 0, len(sinks)) + for _, sink := range sinks { + if sink.AcceptsSignal(signal) { + out = append(out, sink) + } + } + return out +} + func (s *service) SourceStatisticsClear(ctx context.Context, d dependencies.SourceRequestScope, payload *api.SourceStatisticsClearPayload) (err error) { // If user is not admin deny access for write token := d.StorageAPIToken() diff --git a/internal/pkg/service/stream/definition/sink_base.go b/internal/pkg/service/stream/definition/sink_base.go index 8751ea2554..e7e01970ae 100644 --- a/internal/pkg/service/stream/definition/sink_base.go +++ b/internal/pkg/service/stream/definition/sink_base.go @@ -18,6 +18,11 @@ type Sink struct { Description string `json:"description,omitempty" validate:"max=4096"` Config configpatch.PatchKVs `json:"config,omitempty"` // see stream/config/config.Patch + // AllowedSignals filters which OTLP signal types this sink accepts. + // Empty means accept all records (HTTP sources and all OTLP signals). + // Valid values: "logs", "metrics", "traces". + AllowedSignals []string `json:"allowedSignals,omitempty" validate:"dive,oneof=logs metrics traces"` + // Sink type specific fields Table *TableSink `json:"table,omitempty" validate:"required_if=Type table"` @@ -26,3 +31,34 @@ type Sink struct { func (t SinkType) String() string { return string(t) } + +// AcceptsSignal returns true when a record carrying the given OTLP signal +// should be dispatched to this sink — convenience wrapper around +// SignalAccepted using the sink's own AllowedSignals. +func (s *Sink) AcceptsSignal(signal string) bool { + return SignalAccepted(s.AllowedSignals, signal) +} + +// SignalAccepted returns true when a record carrying the given OTLP signal +// should be dispatched to a sink with the given AllowedSignals filter. +// +// The empty signal (HTTP-source records) bypasses the filter so HTTP sources +// ignore AllowedSignals as documented, even when a sink is shared with OTLP. +// An empty AllowedSignals on an OTLP record accepts every signal. +// +// This is the single source of truth used by the runtime router and the +// /test endpoint — keep both paths in sync by routing through here. +func SignalAccepted(allowedSignals []string, signal string) bool { + if signal == "" { + return true + } + if len(allowedSignals) == 0 { + return true + } + for _, allowed := range allowedSignals { + if allowed == signal { + return true + } + } + return false +} diff --git a/internal/pkg/service/stream/definition/source_base.go b/internal/pkg/service/stream/definition/source_base.go index 014a202084..9c65adcdea 100644 --- a/internal/pkg/service/stream/definition/source_base.go +++ b/internal/pkg/service/stream/definition/source_base.go @@ -23,6 +23,7 @@ type Source struct { // Source type specific fields HTTP *HTTPSource `json:"http,omitempty" validate:"required_if=Type http"` + OTLP *OTLPSource `json:"otlp,omitempty" validate:"required_if=Type otlp"` } func (s *Source) FormatHTTPSourceURL(httpSourcePublicURL string) (string, error) { diff --git a/internal/pkg/service/stream/definition/source_otlp.go b/internal/pkg/service/stream/definition/source_otlp.go new file mode 100644 index 0000000000..0e1af4d743 --- /dev/null +++ b/internal/pkg/service/stream/definition/source_otlp.go @@ -0,0 +1,38 @@ +package definition + +import "net/url" + +const ( + SourceTypeOTLP = SourceType("otlp") + + // OTLP signal names — used in AllowedSignals sink filter. + OTLPSignalLogs = "logs" + OTLPSignalMetrics = "metrics" + OTLPSignalTraces = "traces" +) + +type OTLPSource struct { + Secret string `json:"secret" validate:"required,len=48"` +} + +// FormatOTLPSourceURL returns the endpoint URL with the secret embedded as the +// last path segment. Suitable as a single-string convenience value for +// SDK configurations that authenticate by URL only. +func (s *Source) FormatOTLPSourceURL(publicURL string) (string, error) { + u, err := url.Parse(publicURL) + if err != nil { + return "", err + } + return u.JoinPath("otlp", s.ProjectID.String(), s.SourceID.String(), s.OTLP.Secret).String(), nil +} + +// FormatOTLPSourceBaseURL returns the endpoint URL without the secret. Used +// together with the secret in an Authorization: Bearer header so the secret +// stays out of access/CDN/APM logs. +func (s *Source) FormatOTLPSourceBaseURL(publicURL string) (string, error) { + u, err := url.Parse(publicURL) + if err != nil { + return "", err + } + return u.JoinPath("otlp", s.ProjectID.String(), s.SourceID.String()).String(), nil +} diff --git a/internal/pkg/service/stream/definition/source_otlp_test.go b/internal/pkg/service/stream/definition/source_otlp_test.go new file mode 100644 index 0000000000..eb69af6806 --- /dev/null +++ b/internal/pkg/service/stream/definition/source_otlp_test.go @@ -0,0 +1,84 @@ +package definition_test + +import ( + "strings" + "testing" + + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/testvalidation" +) + +func TestOTLPSource_Validation(t *testing.T) { + t.Parallel() + + sourceKey := key.SourceKey{ + BranchKey: key.BranchKey{ProjectID: 123, BranchID: 456}, + SourceID: "my-source", + } + validSecret := strings.Repeat("0", 48) + + cases := testvalidation.TestCases[definition.Source]{ + { + Name: "nil OTLP section", + ExpectedError: `"otlp" is a required field`, + Value: definition.Source{ + SourceKey: sourceKey, + Created: test.Created(), + Versioned: test.Versioned(), + SoftDeletable: test.SoftDeletable(), + Type: definition.SourceTypeOTLP, + Name: "My Source", + Description: "My Description", + }, + }, + { + Name: "empty OTLP section", + ExpectedError: `"otlp.secret" is a required field`, + Value: definition.Source{ + SourceKey: sourceKey, + Created: test.Created(), + Versioned: test.Versioned(), + SoftDeletable: test.SoftDeletable(), + Type: definition.SourceTypeOTLP, + Name: "My Source", + Description: "My Description", + OTLP: &definition.OTLPSource{}, + }, + }, + { + Name: "short secret", + ExpectedError: `"otlp.secret" must be 48 characters in length`, + Value: definition.Source{ + SourceKey: sourceKey, + Created: test.Created(), + Versioned: test.Versioned(), + SoftDeletable: test.SoftDeletable(), + Type: definition.SourceTypeOTLP, + Name: "My Source", + Description: "My Description", + OTLP: &definition.OTLPSource{ + Secret: "tooshort", + }, + }, + }, + { + Name: "ok", + Value: definition.Source{ + SourceKey: sourceKey, + Created: test.Created(), + Versioned: test.Versioned(), + SoftDeletable: test.SoftDeletable(), + Type: definition.SourceTypeOTLP, + Name: "My Source", + Description: "My Description", + OTLP: &definition.OTLPSource{ + Secret: validSecret, + }, + }, + }, + } + + cases.Run(t) +} diff --git a/internal/pkg/service/stream/mapping/recordctx/fasthttp.go b/internal/pkg/service/stream/mapping/recordctx/fasthttp.go index de25a456af..583592b286 100644 --- a/internal/pkg/service/stream/mapping/recordctx/fasthttp.go +++ b/internal/pkg/service/stream/mapping/recordctx/fasthttp.go @@ -46,6 +46,10 @@ func (c *fastHTTPContext) Timestamp() time.Time { return c.timestamp } +func (c *fastHTTPContext) Signal() string { + return "" +} + func (c *fastHTTPContext) ClientIP() net.IP { c.lock.Lock() defer c.lock.Unlock() diff --git a/internal/pkg/service/stream/mapping/recordctx/http.go b/internal/pkg/service/stream/mapping/recordctx/http.go index 0decb3e3f5..44400d2330 100644 --- a/internal/pkg/service/stream/mapping/recordctx/http.go +++ b/internal/pkg/service/stream/mapping/recordctx/http.go @@ -48,6 +48,10 @@ func (c *httpContext) Timestamp() time.Time { return c.timestamp } +func (c *httpContext) Signal() string { + return "" +} + func (c *httpContext) ClientIP() net.IP { c.lock.Lock() defer c.lock.Unlock() diff --git a/internal/pkg/service/stream/mapping/recordctx/otlp.go b/internal/pkg/service/stream/mapping/recordctx/otlp.go index 9babd39266..0b1240b741 100644 --- a/internal/pkg/service/stream/mapping/recordctx/otlp.go +++ b/internal/pkg/service/stream/mapping/recordctx/otlp.go @@ -2,6 +2,7 @@ package recordctx import ( "context" + "io" "net" "net/http" "sort" @@ -21,6 +22,7 @@ type otlpContext struct { clientIP net.IP headers *orderedmap.OrderedMap bodyMap *orderedmap.OrderedMap + signal string lock sync.Mutex headersString *string @@ -45,6 +47,7 @@ func FromOTLP( clientIP net.IP, headers *orderedmap.OrderedMap, bodyMap *orderedmap.OrderedMap, + signal string, ) Context { return &otlpContext{ ctx: ctx, @@ -52,9 +55,50 @@ func FromOTLP( clientIP: clientIP, headers: headers, bodyMap: bodyMap, + signal: signal, } } +// FromOTLPTestRequest builds a test Context for an OTLP source from a plain +// HTTP request. The body must be a JSON object matching the flat record shape +// produced by FlattenLogs/FlattenMetrics/FlattenTraces, so the /test endpoint +// can validate column templates against a representative sample payload. +// The caller is responsible for validating signal (the Goa enum on +// TestSourcePayload.Signal does that at the API boundary); pass "logs" as the +// documented default when the field is omitted. +func FromOTLPTestRequest(ctx context.Context, now time.Time, req *http.Request, signal string) (Context, error) { + bodyBytes := make([]byte, 0) + if req.Body != nil { + var err error + bodyBytes, err = io.ReadAll(req.Body) + if err != nil { + return nil, errors.PrefixError(err, "cannot read request body") + } + } + + bodyMap := orderedmap.New() + if len(bodyBytes) > 0 { + if err := json.Unmarshal(bodyBytes, &bodyMap); err != nil { + return nil, errors.PrefixError(err, "request body must be a JSON object representing a flat OTLP record") + } + } + + headers := orderedmap.New() + for k, v := range req.Header { + if len(v) > 0 { + headers.Set(http.CanonicalHeaderKey(k), v[0]) + } + } + headers.SortKeys(func(keys []string) { sort.Strings(keys) }) + + var clientIP net.IP + if host, _, err := net.SplitHostPort(req.RemoteAddr); err == nil { + clientIP = net.ParseIP(host) + } + + return FromOTLP(ctx, now, clientIP, headers, bodyMap, signal), nil +} + func (c *otlpContext) Ctx() context.Context { return c.ctx } @@ -67,6 +111,10 @@ func (c *otlpContext) ClientIP() net.IP { return c.clientIP } +func (c *otlpContext) Signal() string { + return c.signal +} + func (c *otlpContext) HeadersString() string { c.lock.Lock() defer c.lock.Unlock() diff --git a/internal/pkg/service/stream/mapping/recordctx/otlp_test.go b/internal/pkg/service/stream/mapping/recordctx/otlp_test.go index 6339b108ad..0c5fcdd9d3 100644 --- a/internal/pkg/service/stream/mapping/recordctx/otlp_test.go +++ b/internal/pkg/service/stream/mapping/recordctx/otlp_test.go @@ -20,7 +20,7 @@ func TestOTLPContext_BodyMap_PassesThroughWithoutParsing(t *testing.T) { body.Set("severity_text", "INFO") body.Set("body", "hello") - c := FromOTLP(context.Background(), time.Now(), net.IPv4(127, 0, 0, 1), nil, body) + c := FromOTLP(context.Background(), time.Now(), net.IPv4(127, 0, 0, 1), nil, body, "logs") gotMap, err := c.BodyMap() require.NoError(t, err) @@ -34,7 +34,7 @@ func TestOTLPContext_BodyBytes_LazyJSONMarshal(t *testing.T) { body.Set("severity_text", "WARN") body.Set("count", 42) - c := FromOTLP(context.Background(), time.Now(), net.IPv4(10, 0, 0, 1), nil, body) + c := FromOTLP(context.Background(), time.Now(), net.IPv4(10, 0, 0, 1), nil, body, "logs") bytesA, err := c.BodyBytes() require.NoError(t, err) @@ -56,7 +56,7 @@ func TestOTLPContext_BodyLength(t *testing.T) { body := orderedmap.New() body.Set("k", "v") - c := FromOTLP(context.Background(), time.Now(), nil, nil, body) + c := FromOTLP(context.Background(), time.Now(), nil, nil, body, "logs") expected, err := c.BodyBytes() require.NoError(t, err) @@ -68,7 +68,7 @@ func TestOTLPContext_JSONValue(t *testing.T) { body := orderedmap.New() body.Set("severity_text", "ERROR") - c := FromOTLP(context.Background(), time.Now(), nil, nil, body) + c := FromOTLP(context.Background(), time.Now(), nil, nil, body, "logs") pool := &fastjson.ParserPool{} v, err := c.JSONValue(pool) @@ -82,16 +82,25 @@ func TestOTLPContext_TimestampAndClientIP(t *testing.T) { now := time.Date(2024, 5, 11, 12, 0, 0, 0, time.UTC) ip := net.IPv4(192, 168, 1, 100) - c := FromOTLP(context.Background(), now, ip, nil, orderedmap.New()) + c := FromOTLP(context.Background(), now, ip, nil, orderedmap.New(), "logs") assert.Equal(t, now, c.Timestamp()) assert.True(t, c.ClientIP().Equal(ip)) } +func TestOTLPContext_Signal(t *testing.T) { + t.Parallel() + + for _, signal := range []string{"logs", "metrics", "traces"} { + c := FromOTLP(context.Background(), time.Now(), nil, nil, orderedmap.New(), signal) + assert.Equal(t, signal, c.Signal()) + } +} + func TestOTLPContext_HeadersMap_NilSafe(t *testing.T) { t.Parallel() - c := FromOTLP(context.Background(), time.Now(), nil, nil, orderedmap.New()) + c := FromOTLP(context.Background(), time.Now(), nil, nil, orderedmap.New(), "logs") m := c.HeadersMap() require.NotNil(t, m) assert.Equal(t, 0, m.Len()) @@ -104,7 +113,7 @@ func TestOTLPContext_HeadersString(t *testing.T) { headers.Set("Content-Type", "application/x-protobuf") headers.Set("User-Agent", "otel-go/1.0") - c := FromOTLP(context.Background(), time.Now(), nil, headers, orderedmap.New()) + c := FromOTLP(context.Background(), time.Now(), nil, headers, orderedmap.New(), "logs") s := c.HeadersString() // Canonical header names, sorted, newline-terminated. @@ -117,7 +126,7 @@ func TestOTLPContext_ReleaseBuffers(t *testing.T) { body := orderedmap.New() body.Set("k", "v") - c := FromOTLP(context.Background(), time.Now(), nil, nil, body) + c := FromOTLP(context.Background(), time.Now(), nil, nil, body, "logs") _, err := c.BodyBytes() require.NoError(t, err) diff --git a/internal/pkg/service/stream/mapping/recordctx/recordctx.go b/internal/pkg/service/stream/mapping/recordctx/recordctx.go index 0523119b5b..8323ad7d76 100644 --- a/internal/pkg/service/stream/mapping/recordctx/recordctx.go +++ b/internal/pkg/service/stream/mapping/recordctx/recordctx.go @@ -14,6 +14,9 @@ type Context interface { Ctx() context.Context Timestamp() time.Time ClientIP() net.IP + // Signal returns the OTLP signal type ("logs", "metrics", "traces") for + // OTLP records, or "" for plain HTTP records. + Signal() string HeadersString() string HeadersMap() *orderedmap.OrderedMap ReleaseBuffers() diff --git a/internal/pkg/service/stream/sink/router/collection.go b/internal/pkg/service/stream/sink/router/collection.go index 03c84d6307..f689dfcc84 100644 --- a/internal/pkg/service/stream/sink/router/collection.go +++ b/internal/pkg/service/stream/sink/router/collection.go @@ -18,9 +18,10 @@ type sourceData struct { } type sinkData struct { - sinkKey key.SinkKey - sinkType definition.SinkType - enabled bool + sinkKey key.SinkKey + sinkType definition.SinkType + enabled bool + allowedSignals []string } func newCollection() *collection { @@ -76,7 +77,12 @@ func (c *collection) addSink(sink definition.Sink) { c.sources[sink.SourceKey] = source } - source.sinks[sink.SinkKey] = &sinkData{sinkKey: sink.SinkKey, sinkType: sink.Type, enabled: sink.IsEnabled()} + source.sinks[sink.SinkKey] = &sinkData{ + sinkKey: sink.SinkKey, + sinkType: sink.Type, + enabled: sink.IsEnabled(), + allowedSignals: sink.AllowedSignals, + } } func (c *collection) deleteSink(sinkKey key.SinkKey) { diff --git a/internal/pkg/service/stream/sink/router/router.go b/internal/pkg/service/stream/sink/router/router.go index a1e7188250..50668886ad 100644 --- a/internal/pkg/service/stream/sink/router/router.go +++ b/internal/pkg/service/stream/sink/router/router.go @@ -219,6 +219,7 @@ func (r *Router) DispatchToSource(sourceKey key.SourceKey, c recordctx.Context) } // Write to sinks in parallel + signal := c.Signal() var lock sync.Mutex var wg sync.WaitGroup for _, sink := range source.sinks { @@ -226,6 +227,10 @@ func (r *Router) DispatchToSource(sourceKey key.SourceKey, c recordctx.Context) continue } + if !definition.SignalAccepted(sink.allowedSignals, signal) { + continue + } + r.wg.Add(1) wg.Go(func() { defer r.wg.Done() diff --git a/internal/pkg/service/stream/sink/router/signal_filter_test.go b/internal/pkg/service/stream/sink/router/signal_filter_test.go new file mode 100644 index 0000000000..942cd7f004 --- /dev/null +++ b/internal/pkg/service/stream/sink/router/signal_filter_test.go @@ -0,0 +1,80 @@ +package router_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" +) + +func TestSignalAccepted(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + allowed []string + signal string + expected bool + }{ + // HTTP source: signal is always empty. The sink filter is ignored so the + // documented "HTTP sources ignore allowedSignals" contract holds even + // when an HTTP source reuses a sink that was configured for OTLP. + { + name: "HTTP record, no filter", + allowed: nil, + signal: "", + expected: true, + }, + { + name: "HTTP record, filter set to logs only", + allowed: []string{definition.OTLPSignalLogs}, + signal: "", + expected: true, + }, + { + name: "HTTP record, filter set to all OTLP signals", + allowed: []string{definition.OTLPSignalLogs, definition.OTLPSignalMetrics, definition.OTLPSignalTraces}, + signal: "", + expected: true, + }, + + // OTLP source: signal carries the OTLP signal type. + { + name: "OTLP logs record, empty filter accepts all", + allowed: nil, + signal: definition.OTLPSignalLogs, + expected: true, + }, + { + name: "OTLP logs record, filter allows logs", + allowed: []string{definition.OTLPSignalLogs}, + signal: definition.OTLPSignalLogs, + expected: true, + }, + { + name: "OTLP logs record, filter allows metrics only", + allowed: []string{definition.OTLPSignalMetrics}, + signal: definition.OTLPSignalLogs, + expected: false, + }, + { + name: "OTLP traces record, filter allows logs and metrics", + allowed: []string{definition.OTLPSignalLogs, definition.OTLPSignalMetrics}, + signal: definition.OTLPSignalTraces, + expected: false, + }, + { + name: "OTLP metrics record, multi-signal filter includes metrics", + allowed: []string{definition.OTLPSignalLogs, definition.OTLPSignalMetrics}, + signal: definition.OTLPSignalMetrics, + expected: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tc.expected, definition.SignalAccepted(tc.allowed, tc.signal)) + }) + } +} diff --git a/internal/pkg/service/stream/source/dispatcher/dispatcher.go b/internal/pkg/service/stream/source/dispatcher/dispatcher.go index 82db5f401a..e181a939e1 100644 --- a/internal/pkg/service/stream/source/dispatcher/dispatcher.go +++ b/internal/pkg/service/stream/source/dispatcher/dispatcher.go @@ -37,9 +37,10 @@ type Dispatcher struct { } type sourceData struct { - sourceKey key.SourceKey - enabled bool - secret string + sourceKey key.SourceKey + sourceType definition.SourceType + enabled bool + secret string } type dependencies interface { @@ -69,15 +70,24 @@ func New(d dependencies, logger log.Logger) (*Dispatcher, error) { return sourceKey(source.SourceKey) }, func(key string, source definition.Source, rawValue *op.KeyValue, oldValue **sourceData) *sourceData { + var secret string + switch source.Type { + case definition.SourceTypeHTTP: + secret = source.HTTP.Secret + case definition.SourceTypeOTLP: + secret = source.OTLP.Secret + } return &sourceData{ - sourceKey: source.SourceKey, - enabled: source.IsEnabled(), - secret: source.HTTP.Secret, + sourceKey: source.SourceKey, + sourceType: source.Type, + enabled: source.IsEnabled(), + secret: secret, } }, ). WithFilter(func(event etcdop.WatchEvent[definition.Source]) bool { - return event.Value.Type == definition.SourceTypeHTTP + t := event.Value.Type + return t == definition.SourceTypeHTTP || t == definition.SourceTypeOTLP }). BuildMirror() if err := <-dp.sources.StartMirroring(ctx, &dp.wg, dp.logger, d.Telemetry(), d.WatchTelemetryInterval()); err != nil { @@ -88,7 +98,23 @@ func New(d dependencies, logger log.Logger) (*Dispatcher, error) { return dp, nil } -func (d *Dispatcher) Dispatch(projectID keboola.ProjectID, sourceID key.SourceID, secret string, c recordctx.Context) (*sinkRouter.SourcesResult, error) { +// ValidateSource checks that projectID/sourceID/secret refer to an active source +// of the given expected type, without dispatching any record. The OTLP handler +// calls it on every request before decoding the body so unauthenticated callers +// get a deterministic 404 instead of consuming decode CPU. +func (d *Dispatcher) ValidateSource(projectID keboola.ProjectID, sourceID key.SourceID, secret string, expectedType definition.SourceType) error { + d.wg.Add(1) + defer d.wg.Done() + + if d.isClosed() { + return ShutdownError{} + } + + _, err := d.lookupSources(projectID, sourceID, secret, expectedType) + return err +} + +func (d *Dispatcher) Dispatch(projectID keboola.ProjectID, sourceID key.SourceID, secret string, expectedType definition.SourceType, c recordctx.Context) (*sinkRouter.SourcesResult, error) { d.wg.Add(1) defer d.wg.Done() @@ -97,12 +123,29 @@ func (d *Dispatcher) Dispatch(projectID keboola.ProjectID, sourceID key.SourceID return nil, ShutdownError{} } + matchedSources, err := d.lookupSources(projectID, sourceID, secret, expectedType) + if err != nil { + return nil, err + } + + // Dispatch to all sources in all branches + return d.sinkRouter.DispatchToSources(matchedSources, c), nil +} + +// lookupSources returns all enabled source keys matching projectID/sourceID/secret +// AND the expected source type — so an HTTP secret cannot be used to authenticate +// to /otlp/... endpoints and vice versa. +func (d *Dispatcher) lookupSources(projectID keboola.ProjectID, sourceID key.SourceID, secret string, expectedType definition.SourceType) ([]key.SourceKey, error) { // Get all relevant sources from all branches disabled := 0 var matchedSources []key.SourceKey d.sources.WalkPrefix(sourceKeyPrefix(projectID, sourceID), func(key string, source *sourceData) (stop bool) { - // Secret is now immutable and should be now same in all branches. - // If in the future we would allow secrete to be regenerated in the main/dev branch, it will still work correctly. + // Secret is now immutable and should be the same in all branches. + // If in the future we would allow secret to be regenerated in the main/dev branch, it will still work correctly. + // The source type must also match — the secret is namespaced by transport. + if source.sourceType != expectedType { + return false + } if source.secret == secret { if source.enabled { matchedSources = append(matchedSources, source.sourceKey) @@ -122,8 +165,7 @@ func (d *Dispatcher) Dispatch(projectID keboola.ProjectID, sourceID key.SourceID } } - // Dispatch to all sources in all branches - return d.sinkRouter.DispatchToSources(matchedSources, c), nil + return matchedSources, nil } func (d *Dispatcher) Close(ctx context.Context) error { diff --git a/internal/pkg/service/stream/source/type/httpsource/httpsource.go b/internal/pkg/service/stream/source/type/httpsource/httpsource.go index 0bb9db38a6..f15eee38c7 100644 --- a/internal/pkg/service/stream/source/type/httpsource/httpsource.go +++ b/internal/pkg/service/stream/source/type/httpsource/httpsource.go @@ -20,6 +20,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/service/common/ctxattr" svcErrors "github.com/keboola/keboola-as-code/internal/pkg/service/common/errors" "github.com/keboola/keboola-as-code/internal/pkg/service/common/servicectx" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" definitionRepo "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/repository" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/mapping/recordctx" @@ -106,7 +107,7 @@ func Start(ctx context.Context, d dependencies, cfg Config) error { recordCtx := recordctx.FromFastHTTP(ctx, d.Clock().Now(), c.RequestCtx) // Dispatch request to all sinks - result, err := dp.Dispatch(keboola.ProjectID(projectIDInt), sourceID, secret, recordCtx) + result, err := dp.Dispatch(keboola.ProjectID(projectIDInt), sourceID, secret, definition.SourceTypeHTTP, recordCtx) if err != nil { // Create an enriched context.Context for logging this specific error event. errorLoggingCtx := ctxattr.ContextWith(ctx, @@ -153,6 +154,10 @@ func Start(ctx context.Context, d dependencies, cfg Config) error { // HTTP source (same server, same dispatcher) — the only new pieces are // route registration, OTLP decoding, record flattening, and OTLP-conformant // response construction. + // + // Two route shapes are registered for each signal: one with the secret in + // the URL path (curl-friendly, but leaks into access/CDN logs) and one + // without (header-auth via `Authorization: Bearer `). otlpHandler := otlpsource.New(ctx, logger, d.Clock(), dp, errorHandler) router.Options("/otlp////v1/logs", otlpHandler.HandleOptions) router.Post("/otlp////v1/logs", otlpHandler.HandleLogs) @@ -160,6 +165,12 @@ func Start(ctx context.Context, d dependencies, cfg Config) error { router.Post("/otlp////v1/metrics", otlpHandler.HandleMetrics) router.Options("/otlp////v1/traces", otlpHandler.HandleOptions) router.Post("/otlp////v1/traces", otlpHandler.HandleTraces) + router.Options("/otlp///v1/logs", otlpHandler.HandleOptions) + router.Post("/otlp///v1/logs", otlpHandler.HandleLogs) + router.Options("/otlp///v1/metrics", otlpHandler.HandleOptions) + router.Post("/otlp///v1/metrics", otlpHandler.HandleMetrics) + router.Options("/otlp///v1/traces", otlpHandler.HandleOptions) + router.Post("/otlp///v1/traces", otlpHandler.HandleTraces) // Prepare HTTP server readBufferSize, err := safecast.Convert[int](cfg.ReadBufferSize.Bytes()) diff --git a/internal/pkg/service/stream/source/type/otlpsource/decode.go b/internal/pkg/service/stream/source/type/otlpsource/decode.go index 1a849a5fe2..3d6b6234df 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/decode.go +++ b/internal/pkg/service/stream/source/type/otlpsource/decode.go @@ -23,19 +23,21 @@ const ( ) const ( - contentTypeProtobuf = "application/x-protobuf" - contentTypeJSON = "application/json" + contentTypeProtobuf = "application/x-protobuf" + contentTypeProtobufAlias = "application/protobuf" // accepted for compatibility + contentTypeJSON = "application/json" ) // DetectEncoding parses the Content-Type header value and returns the matching -// OTLP wire format. Unknown values yield EncodingUnsupported. +// OTLP wire format. Media-type comparison is case-insensitive per RFC 9110. +// Unknown values yield EncodingUnsupported. func DetectEncoding(contentType string) Encoding { contentType = strings.TrimSpace(contentType) if i := strings.IndexByte(contentType, ';'); i >= 0 { contentType = strings.TrimSpace(contentType[:i]) } - switch contentType { - case contentTypeProtobuf: + switch strings.ToLower(contentType) { + case contentTypeProtobuf, contentTypeProtobufAlias: return EncodingProtobuf case contentTypeJSON: return EncodingJSON diff --git a/internal/pkg/service/stream/source/type/otlpsource/decode_test.go b/internal/pkg/service/stream/source/type/otlpsource/decode_test.go index 4e81672822..d3d520d0de 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/decode_test.go +++ b/internal/pkg/service/stream/source/type/otlpsource/decode_test.go @@ -26,6 +26,14 @@ func TestDetectEncoding(t *testing.T) { {"application/json; charset=utf-8", EncodingJSON}, {"text/plain", EncodingUnsupported}, {"", EncodingUnsupported}, + // application/protobuf is the OTLP spec alias for application/x-protobuf. + {"application/protobuf", EncodingProtobuf}, + {"application/protobuf; charset=utf-8", EncodingProtobuf}, + // Media types are case-insensitive per RFC 9110. + {"Application/Protobuf", EncodingProtobuf}, + {"APPLICATION/X-PROTOBUF", EncodingProtobuf}, + {"Application/JSON", EncodingJSON}, + {"APPLICATION/JSON", EncodingJSON}, } for _, c := range cases { assert.Equal(t, c.want, DetectEncoding(c.in), "input=%q", c.in) diff --git a/internal/pkg/service/stream/source/type/otlpsource/dispatch.go b/internal/pkg/service/stream/source/type/otlpsource/dispatch.go index 622ccecaaf..901e93d766 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/dispatch.go +++ b/internal/pkg/service/stream/source/type/otlpsource/dispatch.go @@ -9,6 +9,7 @@ import ( "github.com/keboola/keboola-sdk-go/v2/pkg/keboola" svcerrors "github.com/keboola/keboola-as-code/internal/pkg/service/common/errors" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/mapping/recordctx" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/source/dispatcher" @@ -44,23 +45,35 @@ func DispatchRecords( projectID keboola.ProjectID, sourceID key.SourceID, secret string, + signal string, records []FlatRecord, ) DispatchResult { result := DispatchResult{Total: len(records)} for _, rec := range records { - recordCtx := recordctx.FromOTLP(ctx, now, clientIP, headers, rec.Body) - _, err := dp.Dispatch(projectID, sourceID, secret, recordCtx) + recordCtx := recordctx.FromOTLP(ctx, now, clientIP, headers, rec.Body, signal) + sinkResult, routingErr := dp.Dispatch(projectID, sourceID, secret, definition.SourceTypeOTLP, recordCtx) recordCtx.ReleaseBuffers() - if err == nil { + + var statusCode int + var firstErr error + switch { + case routingErr != nil: + // Authentication / routing error (not found, disabled, shutdown). + firstErr = routingErr + statusCode = statusCodeFromError(routingErr) + case sinkResult != nil && sinkResult.StatusCode >= 300: + // One or more sink writes failed; treat the record as rejected. + statusCode = sinkResult.StatusCode + firstErr = errors.Errorf("sink write failed with status %d", statusCode) + default: continue } result.Rejected++ if result.FirstError == nil { - result.FirstError = err + result.FirstError = firstErr } - statusCode := statusCodeFromError(err) if statusCode > result.WorstStatusCode { result.WorstStatusCode = statusCode } diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_common.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_common.go index 9c34c564dc..e40c29e4ef 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_common.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_common.go @@ -2,12 +2,26 @@ package otlpsource import ( "encoding/base64" + "math" "time" "github.com/keboola/go-utils/pkg/orderedmap" "go.opentelemetry.io/collector/pdata/pcommon" ) +// uint64ToInt64Saturating converts a uint64 to int64 by clamping any value +// above math.MaxInt64 to math.MaxInt64. OTLP cumulative counts (histogram / +// summary / data-point counts) are unsigned 64-bit and could theoretically +// exceed the signed range; in practice no production metric will, but +// silently wrapping into negative numbers would corrupt the flattened record. +// Saturation keeps the value as close to truth as the signed encoding allows. +func uint64ToInt64Saturating(v uint64) int64 { + if v > math.MaxInt64 { + return math.MaxInt64 + } + return int64(v) +} + // FlatRecord is a single flattened OTLP record (one log record, one metric data // point, or one span) ready to be wrapped in a recordctx.Context. type FlatRecord struct { @@ -46,7 +60,7 @@ func anyValueToInterface(v pcommon.Value) any { case pcommon.ValueTypeSlice: slice := v.Slice() result := make([]any, slice.Len()) - for i := 0; i < slice.Len(); i++ { + for i := range slice.Len() { result[i] = anyValueToInterface(slice.At(i)) } return result @@ -77,3 +91,24 @@ func makeScopeMap(scope pcommon.InstrumentationScope) *orderedmap.OrderedMap { m.Set("version", scope.Version()) return m } + +// uint64SliceToAny converts []uint64 to []any holding int64 values. +// go-jsonnet's jsonToValue only accepts []interface{} for slices, and +// only handles signed integer types — uint32/uint64 hit the default +// "Not a json type" branch. Values above math.MaxInt64 are saturated. +func uint64SliceToAny(s []uint64) []any { + out := make([]any, len(s)) + for i, v := range s { + out[i] = uint64ToInt64Saturating(v) + } + return out +} + +// float64SliceToAny converts []float64 to []any for the same reason. +func float64SliceToAny(s []float64) []any { + out := make([]any, len(s)) + for i, v := range s { + out[i] = v + } + return out +} diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_logs.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_logs.go index ba71d668c7..bdba1bbf3b 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_logs.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_logs.go @@ -15,15 +15,15 @@ import ( func FlattenLogs(logs plog.Logs) []FlatRecord { records := make([]FlatRecord, 0, logs.LogRecordCount()) - for i := 0; i < logs.ResourceLogs().Len(); i++ { + for i := range logs.ResourceLogs().Len() { rl := logs.ResourceLogs().At(i) resourceAttrs := attributesToMap(rl.Resource().Attributes()) - for j := 0; j < rl.ScopeLogs().Len(); j++ { + for j := range rl.ScopeLogs().Len() { sl := rl.ScopeLogs().At(j) scopeMap := makeScopeMap(sl.Scope()) - for k := 0; k < sl.LogRecords().Len(); k++ { + for k := range sl.LogRecords().Len() { lr := sl.LogRecords().At(k) records = append(records, FlatRecord{Body: flattenLogRecord(lr, resourceAttrs, scopeMap)}) } @@ -41,10 +41,10 @@ func flattenLogRecord( rec := orderedmap.New() rec.Set("timestamp", formatTimestamp(lr.Timestamp())) rec.Set("observed_timestamp", formatTimestamp(lr.ObservedTimestamp())) - rec.Set("severity_number", int32(lr.SeverityNumber())) + rec.Set("severity_number", int64(lr.SeverityNumber())) rec.Set("severity_text", lr.SeverityText()) rec.Set("body", anyValueToInterface(lr.Body())) - rec.Set("flags", uint32(lr.Flags())) + rec.Set("flags", int64(lr.Flags())) // trace_id and span_id are omitted (not "") when empty — the Path column // will fall back to its defaultValue. Emitting "" would mask the absence. diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_logs_test.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_logs_test.go index 0572affd00..4d6f4a4ad3 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_logs_test.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_logs_test.go @@ -44,7 +44,7 @@ func TestFlattenLogs_SingleRecord(t *testing.T) { assert.Equal(t, "2024-01-15T10:30:00Z", getString(t, rec, "timestamp")) assert.Equal(t, "INFO", getString(t, rec, "severity_text")) assert.Equal(t, "User logged in", getValue(t, rec, "body")) - assert.Equal(t, int32(plog.SeverityNumberInfo), getValue(t, rec, "severity_number")) + assert.Equal(t, int64(plog.SeverityNumberInfo), getValue(t, rec, "severity_number")) // attributes / resource / scope are nested ordered maps attrs, _ := rec.Get("attributes") @@ -71,13 +71,13 @@ func TestFlattenLogs_CombinatorialExplosion(t *testing.T) { // 2 resources × 2 scopes × 3 records = 12 flat records. logs := plog.NewLogs() - for r := 0; r < 2; r++ { + for r := range 2 { rl := logs.ResourceLogs().AppendEmpty() rl.Resource().Attributes().PutInt("resource.idx", int64(r)) - for s := 0; s < 2; s++ { + for s := range 2 { sl := rl.ScopeLogs().AppendEmpty() sl.Scope().SetName("scope-" + string(rune('a'+s))) - for k := 0; k < 3; k++ { + for k := range 3 { lr := sl.LogRecords().AppendEmpty() lr.Body().SetStr("rec") lr.Attributes().PutInt("k", int64(k)) diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics.go index 1782b4063d..a7928c62d4 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics.go @@ -16,15 +16,15 @@ import ( func FlattenMetrics(metrics pmetric.Metrics) []FlatRecord { records := make([]FlatRecord, 0, metrics.DataPointCount()) - for i := 0; i < metrics.ResourceMetrics().Len(); i++ { + for i := range metrics.ResourceMetrics().Len() { rm := metrics.ResourceMetrics().At(i) resourceAttrs := attributesToMap(rm.Resource().Attributes()) - for j := 0; j < rm.ScopeMetrics().Len(); j++ { + for j := range rm.ScopeMetrics().Len() { sm := rm.ScopeMetrics().At(j) scopeMap := makeScopeMap(sm.Scope()) - for k := 0; k < sm.Metrics().Len(); k++ { + for k := range sm.Metrics().Len() { m := sm.Metrics().At(k) records = append(records, flattenMetric(m, resourceAttrs, scopeMap)...) } @@ -69,7 +69,7 @@ func flattenNumberPoints( extras func(rec *orderedmap.OrderedMap), ) []FlatRecord { out := make([]FlatRecord, 0, dps.Len()) - for i := 0; i < dps.Len(); i++ { + for i := range dps.Len() { pt := dps.At(i) rec := newMetricBase(name, desc, unit, metricType, resourceAttrs, scopeMap) setDPTimestamps(rec, pt.StartTimestamp(), pt.Timestamp()) @@ -90,12 +90,12 @@ func flattenHistogramPoints( ) []FlatRecord { dps := h.DataPoints() out := make([]FlatRecord, 0, dps.Len()) - for i := 0; i < dps.Len(); i++ { + for i := range dps.Len() { pt := dps.At(i) rec := newMetricBase(name, desc, unit, "histogram", resourceAttrs, scopeMap) setDPTimestamps(rec, pt.StartTimestamp(), pt.Timestamp()) rec.Set("attributes", attributesToMap(pt.Attributes())) - rec.Set("count", pt.Count()) + rec.Set("count", uint64ToInt64Saturating(pt.Count())) if pt.HasSum() { rec.Set("sum", pt.Sum()) } @@ -105,8 +105,8 @@ func flattenHistogramPoints( if pt.HasMax() { rec.Set("max", pt.Max()) } - rec.Set("bucket_counts", pt.BucketCounts().AsRaw()) - rec.Set("explicit_bounds", pt.ExplicitBounds().AsRaw()) + rec.Set("bucket_counts", uint64SliceToAny(pt.BucketCounts().AsRaw())) + rec.Set("explicit_bounds", float64SliceToAny(pt.ExplicitBounds().AsRaw())) rec.Set("aggregation_temporality", h.AggregationTemporality().String()) out = append(out, FlatRecord{Body: rec}) } @@ -120,12 +120,12 @@ func flattenExpHistogramPoints( ) []FlatRecord { dps := h.DataPoints() out := make([]FlatRecord, 0, dps.Len()) - for i := 0; i < dps.Len(); i++ { + for i := range dps.Len() { pt := dps.At(i) rec := newMetricBase(name, desc, unit, "exponential_histogram", resourceAttrs, scopeMap) setDPTimestamps(rec, pt.StartTimestamp(), pt.Timestamp()) rec.Set("attributes", attributesToMap(pt.Attributes())) - rec.Set("count", pt.Count()) + rec.Set("count", uint64ToInt64Saturating(pt.Count())) if pt.HasSum() { rec.Set("sum", pt.Sum()) } @@ -135,8 +135,8 @@ func flattenExpHistogramPoints( if pt.HasMax() { rec.Set("max", pt.Max()) } - rec.Set("scale", pt.Scale()) - rec.Set("zero_count", pt.ZeroCount()) + rec.Set("scale", int64(pt.Scale())) + rec.Set("zero_count", uint64ToInt64Saturating(pt.ZeroCount())) rec.Set("aggregation_temporality", h.AggregationTemporality().String()) out = append(out, FlatRecord{Body: rec}) } @@ -150,17 +150,17 @@ func flattenSummaryPoints( ) []FlatRecord { dps := s.DataPoints() out := make([]FlatRecord, 0, dps.Len()) - for i := 0; i < dps.Len(); i++ { + for i := range dps.Len() { pt := dps.At(i) rec := newMetricBase(name, desc, unit, "summary", resourceAttrs, scopeMap) setDPTimestamps(rec, pt.StartTimestamp(), pt.Timestamp()) rec.Set("attributes", attributesToMap(pt.Attributes())) - rec.Set("count", pt.Count()) + rec.Set("count", uint64ToInt64Saturating(pt.Count())) rec.Set("sum", pt.Sum()) qvSlice := pt.QuantileValues() quantiles := make([]any, 0, qvSlice.Len()) - for q := 0; q < qvSlice.Len(); q++ { + for q := range qvSlice.Len() { qp := qvSlice.At(q) entry := orderedmap.New() entry.Set("quantile", qp.Quantile()) diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics_test.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics_test.go index 02dc7bb4db..042c4d5f2e 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics_test.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_metrics_test.go @@ -86,12 +86,12 @@ func TestFlattenMetrics_Histogram_AllFields(t *testing.T) { require.Len(t, records, 1) body := records[0].Body assert.Equal(t, "histogram", mustGet(t, body, "metric_type")) - assert.Equal(t, uint64(150), mustGet(t, body, "count")) + assert.Equal(t, int64(150), mustGet(t, body, "count")) assert.InDelta(t, 4523.7, mustGet(t, body, "sum"), 1e-9) assert.InDelta(t, 1.2, mustGet(t, body, "min"), 1e-9) assert.InDelta(t, 892.1, mustGet(t, body, "max"), 1e-9) - assert.Equal(t, []uint64{10, 50, 60, 20, 10}, mustGet(t, body, "bucket_counts")) - assert.Equal(t, []float64{5, 25, 50, 100}, mustGet(t, body, "explicit_bounds")) + assert.Equal(t, []any{int64(10), int64(50), int64(60), int64(20), int64(10)}, mustGet(t, body, "bucket_counts")) + assert.Equal(t, []any{float64(5), float64(25), float64(50), float64(100)}, mustGet(t, body, "explicit_bounds")) assert.Equal(t, "Delta", mustGet(t, body, "aggregation_temporality")) } @@ -132,10 +132,10 @@ func TestFlattenMetrics_ExponentialHistogram(t *testing.T) { require.Len(t, records, 1) body := records[0].Body assert.Equal(t, "exponential_histogram", mustGet(t, body, "metric_type")) - assert.Equal(t, uint64(200), mustGet(t, body, "count")) + assert.Equal(t, int64(200), mustGet(t, body, "count")) assert.InDelta(t, 123.4, mustGet(t, body, "sum"), 1e-9) - assert.Equal(t, int32(3), mustGet(t, body, "scale")) - assert.Equal(t, uint64(5), mustGet(t, body, "zero_count")) + assert.Equal(t, int64(3), mustGet(t, body, "scale")) + assert.Equal(t, int64(5), mustGet(t, body, "zero_count")) } func TestFlattenMetrics_Summary_Quantiles(t *testing.T) { diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_traces.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_traces.go index 66547dabef..880ddc6e11 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_traces.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_traces.go @@ -16,15 +16,15 @@ import ( func FlattenTraces(traces ptrace.Traces) []FlatRecord { records := make([]FlatRecord, 0, traces.SpanCount()) - for i := 0; i < traces.ResourceSpans().Len(); i++ { + for i := range traces.ResourceSpans().Len() { rs := traces.ResourceSpans().At(i) resourceAttrs := attributesToMap(rs.Resource().Attributes()) - for j := 0; j < rs.ScopeSpans().Len(); j++ { + for j := range rs.ScopeSpans().Len() { ss := rs.ScopeSpans().At(j) scopeMap := makeScopeMap(ss.Scope()) - for k := 0; k < ss.Spans().Len(); k++ { + for k := range ss.Spans().Len() { span := ss.Spans().At(k) records = append(records, FlatRecord{Body: flattenSpan(span, resourceAttrs, scopeMap)}) } @@ -52,7 +52,7 @@ func flattenSpan( rec.Set("trace_state", span.TraceState().AsRaw()) rec.Set("name", span.Name()) rec.Set("kind", span.Kind().String()) - rec.Set("flags", span.Flags()) + rec.Set("flags", int64(span.Flags())) rec.Set("status_code", span.Status().Code().String()) rec.Set("status_message", span.Status().Message()) rec.Set("attributes", attributesToMap(span.Attributes())) @@ -65,7 +65,7 @@ func flattenSpan( func flattenSpanEvents(events ptrace.SpanEventSlice) []any { out := make([]any, 0, events.Len()) - for i := 0; i < events.Len(); i++ { + for i := range events.Len() { e := events.At(i) entry := orderedmap.New() entry.Set("timestamp", formatTimestamp(e.Timestamp())) @@ -78,7 +78,7 @@ func flattenSpanEvents(events ptrace.SpanEventSlice) []any { func flattenSpanLinks(links ptrace.SpanLinkSlice) []any { out := make([]any, 0, links.Len()) - for i := 0; i < links.Len(); i++ { + for i := range links.Len() { l := links.At(i) entry := orderedmap.New() entry.Set("trace_id", l.TraceID().String()) diff --git a/internal/pkg/service/stream/source/type/otlpsource/flatten_traces_test.go b/internal/pkg/service/stream/source/type/otlpsource/flatten_traces_test.go index 5ea1cfd8c7..250ce17264 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/flatten_traces_test.go +++ b/internal/pkg/service/stream/source/type/otlpsource/flatten_traces_test.go @@ -129,11 +129,11 @@ func TestFlattenTraces_CombinatorialExplosion(t *testing.T) { t.Parallel() traces := ptrace.NewTraces() - for r := 0; r < 2; r++ { + for range 2 { rs := traces.ResourceSpans().AppendEmpty() - for s := 0; s < 3; s++ { + for range 3 { ss := rs.ScopeSpans().AppendEmpty() - for k := 0; k < 4; k++ { + for range 4 { ss.Spans().AppendEmpty().SetName("span") } } diff --git a/internal/pkg/service/stream/source/type/otlpsource/handler.go b/internal/pkg/service/stream/source/type/otlpsource/handler.go index cdb1b45d06..54f4319aff 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/handler.go +++ b/internal/pkg/service/stream/source/type/otlpsource/handler.go @@ -5,6 +5,7 @@ import ( "net/http" "sort" "strconv" + "strings" "github.com/jonboulle/clockwork" "github.com/keboola/go-utils/pkg/orderedmap" @@ -14,6 +15,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/log" svcerrors "github.com/keboola/keboola-as-code/internal/pkg/service/common/errors" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/source/dispatcher" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" @@ -71,21 +73,21 @@ type responseBuilder func(enc Encoding, result DispatchResult) (EncodedResponse, // source. This matches OTLP retry semantics: 4xx means "do not retry", // 5xx means "retry the whole batch". func (h *Handler) HandleLogs(c *routing.Context) error { - return h.handle(c, "logs", decodeAndFlattenLogs, BuildLogsResponse) + return h.handle(c, definition.OTLPSignalLogs, decodeAndFlattenLogs, BuildLogsResponse) } // HandleMetrics serves POST /v1/metrics. One OTLP request typically carries // many metrics, each with many data points; flatten emits one record per // data point, so a single request can dispatch hundreds. func (h *Handler) HandleMetrics(c *routing.Context) error { - return h.handle(c, "metrics", decodeAndFlattenMetrics, BuildMetricsResponse) + return h.handle(c, definition.OTLPSignalMetrics, decodeAndFlattenMetrics, BuildMetricsResponse) } // HandleTraces serves POST /v1/traces. Span events and links are kept nested // under the span rather than exploded into separate records — they are // intrinsically attached to their parent span. func (h *Handler) HandleTraces(c *routing.Context) error { - return h.handle(c, "traces", decodeAndFlattenTraces, BuildTracesResponse) + return h.handle(c, definition.OTLPSignalTraces, decodeAndFlattenTraces, BuildTracesResponse) } func (h *Handler) handle(c *routing.Context, signal string, decode signalDecoder, build responseBuilder) error { @@ -95,10 +97,18 @@ func (h *Handler) handle(c *routing.Context, signal string, decode signalDecoder return nil //nolint:nilerr } + // Authenticate before decompressing/decoding the body so unauthenticated + // callers cannot consume decode CPU and so a bad secret consistently + // returns 404 instead of a parse-error 400/415. + if err := h.dispatcher.ValidateSource(projectID, sourceID, secret, definition.SourceTypeOTLP); err != nil { + h.errorHandler(c.RequestCtx, err) + return nil //nolint:nilerr + } + enc := DetectEncoding(string(c.Request.Header.Peek("Content-Type"))) if enc == EncodingUnsupported { h.errorHandler(c.RequestCtx, svcerrors.NewUnsupportedMediaTypeError( - errors.New(`unsupported OTLP content type, expected "application/x-protobuf" or "application/json"`), + errors.New(`unsupported OTLP content type, expected "application/x-protobuf", "application/protobuf", or "application/json"`), )) return nil } @@ -117,7 +127,7 @@ func (h *Handler) handle(c *routing.Context, signal string, decode signalDecoder return nil //nolint:nilerr } - // Empty batches are valid per the OTLP spec — return 200 immediately. + // Empty batches are valid per the OTLP spec. if len(records) == 0 { h.writeEmptySuccess(c, enc, build) return nil @@ -129,11 +139,12 @@ func (h *Handler) handle(c *routing.Context, signal string, decode signalDecoder ctx, h.dispatcher, h.clock.Now(), - c.RequestCtx.RemoteIP(), + c.RemoteIP(), headers, projectID, sourceID, secret, + signal, records, ) @@ -147,6 +158,10 @@ func (h *Handler) handle(c *routing.Context, signal string, decode signalDecoder if encoded.ContentType != "" { c.Response.Header.Set("Content-Type", encoded.ContentType) } + // Per OTLP spec: SHOULD include Retry-After when returning 429. + if encoded.StatusCode == http.StatusTooManyRequests { + c.Response.Header.Set("Retry-After", "1") + } c.Response.SetBody(encoded.Body) return nil } @@ -204,14 +219,33 @@ func parseAuthParams(c *routing.Context) (keboola.ProjectID, key.SourceID, strin if err != nil { return 0, "", "", svcerrors.NewBadRequestError(errors.Errorf("invalid project ID %q", projectIDStr)) } - return keboola.ProjectID(projectIDInt), key.SourceID(c.Param("sourceID")), c.Param("secret"), nil + // Accept the secret either in the URL path (legacy/curl-friendly form) or in + // the Authorization: Bearer header. The header form keeps the secret out of + // access logs, CDN logs, and APM URL attributes. + secret := c.Param("secret") + if secret == "" { + // HTTP auth scheme is case-insensitive per RFC 9110: accept Bearer, + // bearer, BEARER, etc. The token after the scheme is the secret. + const prefixLen = len("Bearer ") + if auth := string(c.Request.Header.Peek("Authorization")); len(auth) > prefixLen && strings.EqualFold(auth[:prefixLen], "Bearer ") { + secret = auth[prefixLen:] + } + } + return keboola.ProjectID(projectIDInt), key.SourceID(c.Param("sourceID")), secret, nil } func headersToOrderedMap(reqCtx *fasthttp.RequestCtx) *orderedmap.OrderedMap { out := orderedmap.New() for _, k := range reqCtx.Request.Header.PeekKeys() { key := string(k) - out.Set(http.CanonicalHeaderKey(key), string(reqCtx.Request.Header.Peek(key))) + canonical := http.CanonicalHeaderKey(key) + // Drop Authorization so the bearer secret cannot reach a column + // rendered via Header()/headers mapping. The secret has already been + // consumed by parseAuthParams. + if canonical == "Authorization" { + continue + } + out.Set(canonical, string(reqCtx.Request.Header.Peek(key))) } out.SortKeys(func(keys []string) { sort.Strings(keys) diff --git a/internal/pkg/service/stream/source/type/otlpsource/otlpsource_test.go b/internal/pkg/service/stream/source/type/otlpsource/otlpsource_test.go new file mode 100644 index 0000000000..b18cd84d1c --- /dev/null +++ b/internal/pkg/service/stream/source/type/otlpsource/otlpsource_test.go @@ -0,0 +1,786 @@ +package otlpsource_test + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/c2h5oh/datasize" + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/ptrace" + logspb "go.opentelemetry.io/proto/otlp/collector/logs/v1" + "google.golang.org/protobuf/proto" + + commonDeps "github.com/keboola/keboola-as-code/internal/pkg/service/common/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/config" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition/key" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/dependencies" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/source/type/httpsource" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test" + "github.com/keboola/keboola-as-code/internal/pkg/service/stream/storage/test/dummy" + "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" + "github.com/keboola/keboola-as-code/internal/pkg/utils/netutils" +) + +var ( + logsProtoMarshaler = &plog.ProtoMarshaler{} + logsJSONMarshaler = &plog.JSONMarshaler{} + metricsProtoMarshaler = &pmetric.ProtoMarshaler{} + tracesProtoMarshaler = &ptrace.ProtoMarshaler{} +) + +type otlpTestState struct { + ctx context.Context + url string + clk *clockwork.FakeClock + d dependencies.ServiceScope + mock dependencies.Mocked + validSecret string + branchKey key.BranchKey + source definition.Source + sinkLogs definition.Sink + sinkMetrics definition.Sink + sinkTraces definition.Sink + sinkAll definition.Sink +} + +//nolint:tparallel +func TestOTLPSource(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + ts := &otlpTestState{} + ts.ctx = ctx + ts.validSecret = strings.Repeat("1", 48) + + port := netutils.FreePortForTest(t) + ts.url = fmt.Sprintf("http://localhost:%d", port) + ts.clk = clockwork.NewFakeClock() + ts.d, ts.mock = dependencies.NewMockedServiceScopeWithConfig(t, ctx, func(cfg *config.Config) { + cfg.Source.HTTP.Listen = fmt.Sprintf("0.0.0.0:%d", port) + cfg.Source.HTTP.MaxRequestBodySize = 1 * datasize.MB + }, commonDeps.WithClock(ts.clk)) + + // Create branch, OTLP source and four sinks with different signal filters. + ts.branchKey = key.BranchKey{ProjectID: 123, BranchID: 111} + branch := test.NewBranch(ts.branchKey) + + ts.source = test.NewOTLPSource(key.SourceKey{BranchKey: ts.branchKey, SourceID: "my-source"}) + ts.source.OTLP.Secret = ts.validSecret + + sinkLogsKey := key.SinkKey{SourceKey: ts.source.SourceKey, SinkID: "sink-logs"} + sinkMetricsKey := key.SinkKey{SourceKey: ts.source.SourceKey, SinkID: "sink-metrics"} + sinkTracesKey := key.SinkKey{SourceKey: ts.source.SourceKey, SinkID: "sink-traces"} + sinkAllKey := key.SinkKey{SourceKey: ts.source.SourceKey, SinkID: "sink-all"} + + ts.sinkLogs = dummy.NewSink(sinkLogsKey) + ts.sinkLogs.AllowedSignals = []string{definition.OTLPSignalLogs} + + ts.sinkMetrics = dummy.NewSink(sinkMetricsKey) + ts.sinkMetrics.AllowedSignals = []string{definition.OTLPSignalMetrics} + + ts.sinkTraces = dummy.NewSink(sinkTracesKey) + ts.sinkTraces.AllowedSignals = []string{definition.OTLPSignalTraces} + + ts.sinkAll = dummy.NewSink(sinkAllKey) + // AllowedSignals empty = accept everything + + repo := ts.d.DefinitionRepository() + require.NoError(t, repo.Branch().Create(&branch, ts.clk.Now(), test.ByUser()).Do(ts.ctx).Err()) + require.NoError(t, repo.Source().Create(&ts.source, ts.clk.Now(), test.ByUser(), "create").Do(ts.ctx).Err()) + require.NoError(t, repo.Sink().Create(&ts.sinkLogs, ts.clk.Now(), test.ByUser(), "create").Do(ts.ctx).Err()) + require.NoError(t, repo.Sink().Create(&ts.sinkMetrics, ts.clk.Now(), test.ByUser(), "create").Do(ts.ctx).Err()) + require.NoError(t, repo.Sink().Create(&ts.sinkTraces, ts.clk.Now(), test.ByUser(), "create").Do(ts.ctx).Err()) + require.NoError(t, repo.Sink().Create(&ts.sinkAll, ts.clk.Now(), test.ByUser(), "create").Do(ts.ctx).Err()) + + require.NoError(t, stream.StartComponents(ts.ctx, ts.d, ts.mock.TestConfig(), stream.ComponentHTTPSource)) + t.Cleanup(func() { + ts.d.Process().Shutdown(ts.ctx, errors.New("bye bye")) + ts.d.Process().WaitForShutdown() + }) + require.NoError(t, netutils.WaitForHTTP(ts.url, 10*time.Second)) + + runOTLPTestCases(t, ts) +} + +func runOTLPTestCases(t *testing.T, ts *otlpTestState) { + t.Helper() + + baseURL := fmt.Sprintf("/otlp/123/my-source/%s", ts.validSecret) + wrongSecret := strings.Repeat("0", 48) + + for _, tc := range []struct { + name string + prepare func(t *testing.T) + method string + path string + headers map[string]string + body []byte + expectedStatusCode int + expectedHeaders map[string]string + expectedBodyJSON string + }{ + // ---- transport / routing errors ------------------------------------------- + + { + name: "OPTIONS CORS preflight", + method: http.MethodOptions, + path: baseURL + "/v1/logs", + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{ + "Allow": "OPTIONS, POST", + "Access-Control-Allow-Methods": "OPTIONS, POST", + "Access-Control-Allow-Headers": "*", + "Access-Control-Allow-Origin": "*", + "Server": httpsource.ServerHeader, + }, + }, + { + name: "unsupported content-type", + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{"Content-Type": "text/plain"}, + body: []byte("hello"), + expectedStatusCode: http.StatusUnsupportedMediaType, + }, + { + name: "invalid project ID", + method: http.MethodPost, + path: "/otlp/foo/my-source/" + ts.validSecret + "/v1/logs", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusBadRequest, + }, + { + name: "source not found - wrong secret", + method: http.MethodPost, + path: "/otlp/123/my-source/" + wrongSecret + "/v1/logs", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusNotFound, + }, + + // ---- header-auth: Authorization: Bearer on the secret-less route ----- + + { + name: "bearer auth - logs success", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "Bearer " + ts.validSecret, + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + }, + { + // RFC 9110 auth schemes are case-insensitive. + name: "bearer auth - lowercase scheme", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "bearer " + ts.validSecret, + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + }, + { + name: "bearer auth - mixed-case scheme", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "BeArEr " + ts.validSecret, + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + }, + { + name: "bearer auth - metrics success", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/metrics", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "Bearer " + ts.validSecret, + }, + body: mustMarshalMetrics(t, sampleMetrics()), + expectedStatusCode: http.StatusOK, + }, + { + name: "bearer auth - traces success", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/traces", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "Bearer " + ts.validSecret, + }, + body: mustMarshalTraces(t, sampleTraces()), + expectedStatusCode: http.StatusOK, + }, + { + name: "bearer auth - wrong secret", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "Bearer " + wrongSecret, + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusNotFound, + }, + { + name: "bearer auth - missing Authorization header", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/logs", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusNotFound, + }, + { + name: "bearer auth - malformed Authorization header", + method: http.MethodPost, + path: "/otlp/123/my-source/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Authorization": "Basic " + ts.validSecret, + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusNotFound, + }, + { + name: "bearer auth - OPTIONS preflight on secret-less route", + method: http.MethodOptions, + path: "/otlp/123/my-source/v1/logs", + expectedHeaders: map[string]string{ + "Allow": "OPTIONS, POST", + "Access-Control-Allow-Methods": "OPTIONS, POST", + "Access-Control-Allow-Origin": "*", + }, + expectedStatusCode: http.StatusOK, + }, + { + name: "source disabled", + prepare: func(t *testing.T) { + t.Helper() + require.NoError(t, ts.d.DefinitionRepository().Source().Disable( + ts.source.SourceKey, ts.clk.Now(), test.ByUser(), "maintenance", + ).Do(ts.ctx).Err()) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + ts.mock.DebugLogger().AssertJSONMessages(c, `{"level":"debug","message":"watch stream mirror synced to revision %s","component":"http-source.dispatcher"}`) + }, 5*time.Second, 50*time.Millisecond) + }, + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusNotFound, + }, + + // ---- re-enable source for remaining tests ---------------------------------- + + { + name: "re-enable source", + prepare: func(t *testing.T) { + t.Helper() + require.NoError(t, ts.d.DefinitionRepository().Source().Enable( + ts.source.SourceKey, ts.clk.Now(), test.ByUser(), + ).Do(ts.ctx).Err()) + assert.EventuallyWithT(t, func(c *assert.CollectT) { + ts.mock.DebugLogger().AssertJSONMessages(c, `{"level":"debug","message":"watch stream mirror synced to revision %s","component":"http-source.dispatcher"}`) + }, 5*time.Second, 50*time.Millisecond) + }, + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + }, + + // ---- empty batches -------------------------------------------------------- + + { + name: "empty logs batch", + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalLogs(t, plog.NewLogs()), + expectedStatusCode: http.StatusOK, + }, + { + name: "empty metrics batch", + method: http.MethodPost, + path: baseURL + "/v1/metrics", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalMetrics(t, pmetric.NewMetrics()), + expectedStatusCode: http.StatusOK, + }, + { + name: "empty traces batch", + method: http.MethodPost, + path: baseURL + "/v1/traces", + headers: map[string]string{"Content-Type": "application/x-protobuf"}, + body: mustMarshalTraces(t, ptrace.NewTraces()), + expectedStatusCode: http.StatusOK, + }, + + // ---- logs ----------------------------------------------------------------- + + { + name: "logs - protobuf", + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "application/x-protobuf"}, + }, + { + name: "logs - application/protobuf alias", + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{ + "Content-Type": "application/protobuf", + }, + body: mustMarshalLogs(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "application/x-protobuf"}, + }, + { + name: "logs - JSON", + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{ + "Content-Type": "application/json", + }, + body: mustMarshalLogsJSON(t, sampleLogs()), + expectedStatusCode: http.StatusOK, + expectedHeaders: map[string]string{"Content-Type": "application/json"}, + }, + { + name: "logs - gzip compressed protobuf", + method: http.MethodPost, + path: baseURL + "/v1/logs", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + "Content-Encoding": "gzip", + }, + body: mustGzip(t, mustMarshalLogs(t, sampleLogs())), + expectedStatusCode: http.StatusOK, + }, + + // ---- metrics -------------------------------------------------------------- + + { + name: "metrics - protobuf", + method: http.MethodPost, + path: baseURL + "/v1/metrics", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + }, + body: mustMarshalMetrics(t, sampleMetrics()), + expectedStatusCode: http.StatusOK, + }, + + // ---- traces --------------------------------------------------------------- + + { + name: "traces - protobuf", + method: http.MethodPost, + path: baseURL + "/v1/traces", + headers: map[string]string{ + "Content-Type": "application/x-protobuf", + }, + body: mustMarshalTraces(t, sampleTraces()), + expectedStatusCode: http.StatusOK, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if tc.prepare != nil { + tc.prepare(t) + } + ts.mock.DebugLogger().Truncate() + + req, err := http.NewRequestWithContext(ts.ctx, tc.method, ts.url+tc.path, bytes.NewReader(tc.body)) + require.NoError(t, err) + for k, v := range tc.headers { + req.Header.Set(k, v) + } + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + + assert.Equal(t, tc.expectedStatusCode, resp.StatusCode, "status code") + + for k, v := range tc.expectedHeaders { + assert.Equal(t, v, resp.Header.Get(k), "header %s", k) + } + + if tc.expectedBodyJSON != "" { + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.JSONEq(t, tc.expectedBodyJSON, string(body)) + } + }) + } +} + +func TestOTLPSource_SignalRouting(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + port := netutils.FreePortForTest(t) + clk := clockwork.NewFakeClock() + d, mock := dependencies.NewMockedServiceScopeWithConfig(t, ctx, func(cfg *config.Config) { + cfg.Source.HTTP.Listen = fmt.Sprintf("0.0.0.0:%d", port) + }, commonDeps.WithClock(clk)) + + validSecret := strings.Repeat("2", 48) + branchKey := key.BranchKey{ProjectID: 456, BranchID: 1} + sourceKey := key.SourceKey{BranchKey: branchKey, SourceID: "routing-source"} + + branch := test.NewBranch(branchKey) + source := test.NewOTLPSource(sourceKey) + source.OTLP.Secret = validSecret + + logsKey := key.SinkKey{SourceKey: sourceKey, SinkID: "logs-sink"} + metricsKey := key.SinkKey{SourceKey: sourceKey, SinkID: "metrics-sink"} + tracesKey := key.SinkKey{SourceKey: sourceKey, SinkID: "traces-sink"} + allKey := key.SinkKey{SourceKey: sourceKey, SinkID: "all-sink"} + + sinkLogs := dummy.NewSink(logsKey) + sinkLogs.AllowedSignals = []string{definition.OTLPSignalLogs} + + sinkMetrics := dummy.NewSink(metricsKey) + sinkMetrics.AllowedSignals = []string{definition.OTLPSignalMetrics} + + sinkTraces := dummy.NewSink(tracesKey) + sinkTraces.AllowedSignals = []string{definition.OTLPSignalTraces} + + // Unfiltered sink — should receive records from every signal endpoint. + sinkAll := dummy.NewSink(allKey) + + repo := d.DefinitionRepository() + require.NoError(t, repo.Branch().Create(&branch, clk.Now(), test.ByUser()).Do(ctx).Err()) + require.NoError(t, repo.Source().Create(&source, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + require.NoError(t, repo.Sink().Create(&sinkLogs, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + require.NoError(t, repo.Sink().Create(&sinkMetrics, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + require.NoError(t, repo.Sink().Create(&sinkTraces, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + require.NoError(t, repo.Sink().Create(&sinkAll, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + + ctrl := mock.TestDummySinkController() + + require.NoError(t, stream.StartComponents(ctx, d, mock.TestConfig(), stream.ComponentHTTPSource)) + t.Cleanup(func() { + d.Process().Shutdown(ctx, errors.New("done")) + d.Process().WaitForShutdown() + }) + + baseURL := fmt.Sprintf("http://localhost:%d/otlp/456/routing-source/%s", port, validSecret) + require.NoError(t, netutils.WaitForHTTP(fmt.Sprintf("http://localhost:%d", port), 10*time.Second)) + + // One log record reaches logs-sink and all-sink only. + ctrl.ResetWriteCounts() + resp := doOTLPPost(t, ctx, baseURL+"/v1/logs", mustMarshalLogs(t, sampleLogs())) + assert.Equal(t, http.StatusOK, resp.StatusCode) + _ = resp.Body.Close() + assert.Equal(t, 1, ctrl.WriteCount(logsKey), "logs record reached logs-sink") + assert.Equal(t, 0, ctrl.WriteCount(metricsKey), "logs record did NOT reach metrics-sink") + assert.Equal(t, 0, ctrl.WriteCount(tracesKey), "logs record did NOT reach traces-sink") + assert.Equal(t, 1, ctrl.WriteCount(allKey), "logs record reached unfiltered all-sink") + + // One metric data point reaches metrics-sink and all-sink only. + ctrl.ResetWriteCounts() + resp = doOTLPPost(t, ctx, baseURL+"/v1/metrics", mustMarshalMetrics(t, sampleMetrics())) + assert.Equal(t, http.StatusOK, resp.StatusCode) + _ = resp.Body.Close() + assert.Equal(t, 0, ctrl.WriteCount(logsKey), "metric did NOT reach logs-sink") + assert.Equal(t, 1, ctrl.WriteCount(metricsKey), "metric reached metrics-sink") + assert.Equal(t, 0, ctrl.WriteCount(tracesKey), "metric did NOT reach traces-sink") + assert.Equal(t, 1, ctrl.WriteCount(allKey), "metric reached unfiltered all-sink") + + // One span reaches traces-sink and all-sink only. + ctrl.ResetWriteCounts() + resp = doOTLPPost(t, ctx, baseURL+"/v1/traces", mustMarshalTraces(t, sampleTraces())) + assert.Equal(t, http.StatusOK, resp.StatusCode) + _ = resp.Body.Close() + assert.Equal(t, 0, ctrl.WriteCount(logsKey), "trace did NOT reach logs-sink") + assert.Equal(t, 0, ctrl.WriteCount(metricsKey), "trace did NOT reach metrics-sink") + assert.Equal(t, 1, ctrl.WriteCount(tracesKey), "trace reached traces-sink") + assert.Equal(t, 1, ctrl.WriteCount(allKey), "trace reached unfiltered all-sink") +} + +func TestOTLPSource_PartialSuccess(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + port := netutils.FreePortForTest(t) + clk := clockwork.NewFakeClock() + d, mock := dependencies.NewMockedServiceScopeWithConfig(t, ctx, func(cfg *config.Config) { + cfg.Source.HTTP.Listen = fmt.Sprintf("0.0.0.0:%d", port) + }, commonDeps.WithClock(clk)) + + validSecret := strings.Repeat("3", 48) + branchKey := key.BranchKey{ProjectID: 789, BranchID: 1} + sourceKey := key.SourceKey{BranchKey: branchKey, SourceID: "partial-source"} + + branch := test.NewBranch(branchKey) + source := test.NewOTLPSource(sourceKey) + source.OTLP.Secret = validSecret + + ctrl := mock.TestDummySinkController() + ctrl.PipelineWriteError = errors.New("disk full") + + sink := dummy.NewSink(key.SinkKey{SourceKey: sourceKey, SinkID: "my-sink"}) + + repo := d.DefinitionRepository() + require.NoError(t, repo.Branch().Create(&branch, clk.Now(), test.ByUser()).Do(ctx).Err()) + require.NoError(t, repo.Source().Create(&source, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + require.NoError(t, repo.Sink().Create(&sink, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + + require.NoError(t, stream.StartComponents(ctx, d, mock.TestConfig(), stream.ComponentHTTPSource)) + t.Cleanup(func() { + d.Process().Shutdown(ctx, errors.New("done")) + d.Process().WaitForShutdown() + }) + require.NoError(t, netutils.WaitForHTTP(fmt.Sprintf("http://localhost:%d", port), 10*time.Second)) + + // All-rejected path: 2 records, every WriteRecord errors out. Per the OTLP + // response builder this escalates to a top-level 5xx instead of a 200 with + // partial_success (see shouldEscalateToError in response.go). The mixed + // accept/reject path is exercised by TestOTLPSource_PartialSuccess_Mixed. + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("service.name", "test-service") + sl := rl.ScopeLogs().AppendEmpty() + lr1 := sl.LogRecords().AppendEmpty() + lr1.Body().SetStr("record one") + lr2 := sl.LogRecords().AppendEmpty() + lr2.Body().SetStr("record two") + + baseURL := fmt.Sprintf("http://localhost:%d/otlp/789/partial-source/%s", port, validSecret) + resp := doOTLPPost(t, ctx, baseURL+"/v1/logs", mustMarshalLogs(t, logs)) + defer func() { _ = resp.Body.Close() }() + + // When all records fail with a retryable error, the handler escalates to a top-level 5xx. + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) +} + +// TestOTLPSource_PartialSuccess_Mixed exercises the actual partial-success +// path: some records succeed and some fail in the same batch, so the response +// is HTTP 200 with an OTLP partial_success body that reports a non-zero +// rejected count. +func TestOTLPSource_PartialSuccess_Mixed(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + port := netutils.FreePortForTest(t) + clk := clockwork.NewFakeClock() + d, mock := dependencies.NewMockedServiceScopeWithConfig(t, ctx, func(cfg *config.Config) { + cfg.Source.HTTP.Listen = fmt.Sprintf("0.0.0.0:%d", port) + }, commonDeps.WithClock(clk)) + + validSecret := strings.Repeat("4", 48) + branchKey := key.BranchKey{ProjectID: 555, BranchID: 1} + sourceKey := key.SourceKey{BranchKey: branchKey, SourceID: "mixed-source"} + + branch := test.NewBranch(branchKey) + source := test.NewOTLPSource(sourceKey) + source.OTLP.Secret = validSecret + + // Hook: every second WriteRecord call returns an error. With a 4-record batch + // the deterministic outcome is 2 accepted, 2 rejected. + ctrl := mock.TestDummySinkController() + var calls atomic.Int64 + ctrl.PipelineWriteHook = func(_ key.SinkKey) error { + if calls.Add(1)%2 == 0 { + return errors.New("transient sink error") + } + return nil + } + + sink := dummy.NewSink(key.SinkKey{SourceKey: sourceKey, SinkID: "mixed-sink"}) + + repo := d.DefinitionRepository() + require.NoError(t, repo.Branch().Create(&branch, clk.Now(), test.ByUser()).Do(ctx).Err()) + require.NoError(t, repo.Source().Create(&source, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + require.NoError(t, repo.Sink().Create(&sink, clk.Now(), test.ByUser(), "create").Do(ctx).Err()) + + require.NoError(t, stream.StartComponents(ctx, d, mock.TestConfig(), stream.ComponentHTTPSource)) + t.Cleanup(func() { + d.Process().Shutdown(ctx, errors.New("done")) + d.Process().WaitForShutdown() + }) + require.NoError(t, netutils.WaitForHTTP(fmt.Sprintf("http://localhost:%d", port), 10*time.Second)) + + // 4-record logs batch — hook alternates accept/fail, so 2 records are rejected. + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("service.name", "mixed-test") + sl := rl.ScopeLogs().AppendEmpty() + for i := range 4 { + lr := sl.LogRecords().AppendEmpty() + lr.Body().SetStr(fmt.Sprintf("record %d", i)) + } + + baseURL := fmt.Sprintf("http://localhost:%d/otlp/555/mixed-source/%s", port, validSecret) + resp := doOTLPPost(t, ctx, baseURL+"/v1/logs", mustMarshalLogs(t, logs)) + defer func() { _ = resp.Body.Close() }() + + // Top-level 200 with an OTLP partial_success body reporting the rejected count. + assert.Equal(t, http.StatusOK, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + parsed := &logspb.ExportLogsServiceResponse{} + require.NoError(t, proto.Unmarshal(body, parsed), "response body must be a valid OTLP logs export response") + require.NotNil(t, parsed.GetPartialSuccess(), "partial_success block must be present") + assert.EqualValues(t, 2, parsed.GetPartialSuccess().GetRejectedLogRecords(), "exactly 2 of 4 records should be rejected") +} + +// ---- sample OTel payload helpers -------------------------------------------- + +func sampleLogs() plog.Logs { + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + rl.Resource().Attributes().PutStr("service.name", "auth-service") + rl.Resource().Attributes().PutStr("deployment.environment", "production") + + sl := rl.ScopeLogs().AppendEmpty() + sl.Scope().SetName("github.com/my/auth") + sl.Scope().SetVersion("1.2.3") + + lr := sl.LogRecords().AppendEmpty() + ts := pcommon.NewTimestampFromTime(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC)) + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberInfo) + lr.SetSeverityText("INFO") + lr.Body().SetStr("User login successful") + lr.Attributes().PutStr("user.id", "u-42") + lr.Attributes().PutStr("request.id", "req-abc123") + return logs +} + +func sampleMetrics() pmetric.Metrics { + metrics := pmetric.NewMetrics() + rm := metrics.ResourceMetrics().AppendEmpty() + rm.Resource().Attributes().PutStr("service.name", "cart-service") + + sm := rm.ScopeMetrics().AppendEmpty() + sm.Scope().SetName("github.com/my/cart") + + m := sm.Metrics().AppendEmpty() + m.SetName("http.server.request.duration") + m.SetDescription("Duration of HTTP server requests.") + m.SetUnit("s") + + hist := m.SetEmptyHistogram() + hist.SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) + + dp := hist.DataPoints().AppendEmpty() + dp.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC))) + dp.SetTimestamp(pcommon.NewTimestampFromTime(time.Date(2024, 6, 1, 12, 1, 0, 0, time.UTC))) + dp.SetCount(100) + dp.SetSum(4.5) + dp.Attributes().PutStr("http.method", "GET") + dp.Attributes().PutStr("http.status_code", "200") + return metrics +} + +func sampleTraces() ptrace.Traces { + traces := ptrace.NewTraces() + rs := traces.ResourceSpans().AppendEmpty() + rs.Resource().Attributes().PutStr("service.name", "order-service") + + ss := rs.ScopeSpans().AppendEmpty() + ss.Scope().SetName("github.com/my/orders") + + span := ss.Spans().AppendEmpty() + span.SetTraceID([16]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) + span.SetSpanID([8]byte{1, 2, 3, 4, 5, 6, 7, 8}) + span.SetName("POST /orders") + span.SetKind(ptrace.SpanKindServer) + span.SetStartTimestamp(pcommon.NewTimestampFromTime(time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC))) + span.SetEndTimestamp(pcommon.NewTimestampFromTime(time.Date(2024, 6, 1, 12, 0, 0, 50_000_000, time.UTC))) + span.Attributes().PutStr("http.method", "POST") + span.Attributes().PutInt("http.status_code", 201) + + event := span.Events().AppendEmpty() + event.SetName("order.created") + event.Attributes().PutStr("order.id", "ord-9999") + return traces +} + +// ---- marshal helpers -------------------------------------------------------- + +func mustMarshalLogs(t *testing.T, logs plog.Logs) []byte { + t.Helper() + b, err := logsProtoMarshaler.MarshalLogs(logs) + require.NoError(t, err) + return b +} + +func mustMarshalLogsJSON(t *testing.T, logs plog.Logs) []byte { + t.Helper() + b, err := logsJSONMarshaler.MarshalLogs(logs) + require.NoError(t, err) + return b +} + +func mustMarshalMetrics(t *testing.T, metrics pmetric.Metrics) []byte { + t.Helper() + b, err := metricsProtoMarshaler.MarshalMetrics(metrics) + require.NoError(t, err) + return b +} + +func mustMarshalTraces(t *testing.T, traces ptrace.Traces) []byte { + t.Helper() + b, err := tracesProtoMarshaler.MarshalTraces(traces) + require.NoError(t, err) + return b +} + +func mustGzip(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + _, err := w.Write(data) + require.NoError(t, err) + require.NoError(t, w.Close()) + return buf.Bytes() +} + +func doOTLPPost(t *testing.T, ctx context.Context, url string, body []byte) *http.Response { + t.Helper() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/x-protobuf") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp +} diff --git a/internal/pkg/service/stream/source/type/otlpsource/response.go b/internal/pkg/service/stream/source/type/otlpsource/response.go index cc4ccd3cc7..2f523c6e3a 100644 --- a/internal/pkg/service/stream/source/type/otlpsource/response.go +++ b/internal/pkg/service/stream/source/type/otlpsource/response.go @@ -25,14 +25,10 @@ type EncodedResponse struct { // // Success vs partial-success vs top-level error: // - All records dispatched OK: 200 with empty ExportLogsServiceResponse. -// - Some records rejected, no top-level failure: 200 with partial_success -// populated. The OTLP client SHOULD NOT retry rejected records. -// - All records failed with the same retryable status (5xx, 429): return -// that status code; the OTLP client SHOULD retry the entire batch. -// -// We choose top-level error when result.Rejected == result.Total AND -// result.WorstStatusCode is retryable (5xx or 429). Otherwise we return 200 -// with partial_success so non-retryable rejections (4xx) are not retried. +// - Some records rejected, rest OK: 200 with partial_success populated. +// - All records rejected: return the worst status code directly, whether 4xx +// (e.g. 404 unknown source) or 5xx/429 (transient failure). 4xx tells the +// client "fix your configuration"; 5xx/429 tells it to retry the batch. func BuildLogsResponse(enc Encoding, result DispatchResult) (EncodedResponse, error) { if shouldEscalateToError(result) { return EncodedResponse{StatusCode: result.WorstStatusCode}, nil @@ -80,12 +76,13 @@ func BuildTracesResponse(enc Encoding, result DispatchResult) (EncodedResponse, return encode(enc, resp) } -// shouldEscalateToError returns true when every record failed AND with a -// retryable status. That signals the client to retry the entire batch. -// Mixed outcomes and non-retryable rejections stay at 200 with partial_success -// so clients do not retry rejected records that will not succeed. +// shouldEscalateToError returns true when every record in the batch was +// rejected. In that case the handler returns the worst HTTP status code +// directly: 4xx for permanent errors (wrong secret, disabled source), 5xx/429 +// for transient ones. Mixed outcomes — at least one record accepted — stay at +// 200 with partial_success. func shouldEscalateToError(r DispatchResult) bool { - return r.Rejected > 0 && r.Rejected == r.Total && isRetryable(r.WorstStatusCode) + return r.Rejected > 0 && r.Rejected == r.Total } func encode(enc Encoding, msg proto.Message) (EncodedResponse, error) { @@ -126,12 +123,3 @@ func formatRejectionMessage(r DispatchResult) string { return strconv.Itoa(r.Rejected) + " of " + strconv.Itoa(r.Total) + " records rejected; first error: " + r.FirstError.Error() } - -// isRetryable maps a Stream error status code to OTLP retry semantics. -// 5xx and 429 are retryable per the OTLP spec. -func isRetryable(statusCode int) bool { - if statusCode == 429 { - return true - } - return statusCode >= 500 && statusCode < 600 -} diff --git a/internal/pkg/service/stream/storage/test/dummy/sink.go b/internal/pkg/service/stream/storage/test/dummy/sink.go index 47a267cb41..10eec82323 100644 --- a/internal/pkg/service/stream/storage/test/dummy/sink.go +++ b/internal/pkg/service/stream/storage/test/dummy/sink.go @@ -2,6 +2,7 @@ package dummy import ( "context" + "sync" "time" "github.com/keboola/keboola-as-code/internal/pkg/service/stream/definition" @@ -31,14 +32,47 @@ type SinkController struct { PipelineOpenError error PipelineWriteRecordStatus pipeline.RecordStatus PipelineWriteError error - UploadHandler func(ctx context.Context, volume *diskreader.Volume, slice plugin.Slice, stats statistics.Value) error - UploadError error - ImportHandler func(ctx context.Context, file plugin.File, stats statistics.Value) error - ImportError error + // PipelineWriteHook overrides PipelineWriteError when non-nil. It is invoked + // for each WriteRecord call so tests can produce per-record outcomes (e.g. + // fail every other record to exercise partial_success paths). + PipelineWriteHook func(sinkKey key.SinkKey) error + UploadHandler func(ctx context.Context, volume *diskreader.Volume, slice plugin.Slice, stats statistics.Value) error + UploadError error + ImportHandler func(ctx context.Context, file plugin.File, stats statistics.Value) error + ImportError error + + // counts tracks how many records were written to each sink so tests can + // assert per-sink routing (which sinks were selected for a given record). + countsMu sync.Mutex + counts map[key.SinkKey]int +} + +// WriteCount returns the number of WriteRecord calls observed for the given sink. +func (c *SinkController) WriteCount(k key.SinkKey) int { + c.countsMu.Lock() + defer c.countsMu.Unlock() + return c.counts[k] +} + +// ResetWriteCounts clears all per-sink counters. +func (c *SinkController) ResetWriteCounts() { + c.countsMu.Lock() + defer c.countsMu.Unlock() + c.counts = nil +} + +func (c *SinkController) recordWrite(k key.SinkKey) { + c.countsMu.Lock() + defer c.countsMu.Unlock() + if c.counts == nil { + c.counts = make(map[key.SinkKey]int) + } + c.counts[k]++ } type Pipeline struct { controller *SinkController + sinkKey key.SinkKey onClose func(ctx context.Context, cause string) } @@ -87,7 +121,7 @@ func (c *SinkController) RegisterDummySinkTypes(plugins *plugin.Plugins, control // Register dummy pipeline opener for tests plugins.RegisterSinkPipelineOpener(func(ctx context.Context, sinkKey key.SinkKey, sinkType definition.SinkType, onClose func(ctx context.Context, cause string)) (pipeline.Pipeline, error) { if sinkType == SinkType { - return controller.OpenPipeline(onClose) + return controller.OpenPipeline(sinkKey, onClose) } return nil, pipeline.NoOpenerFoundError{SinkType: sinkType} @@ -115,11 +149,11 @@ func (c *SinkController) RegisterDummySinkTypes(plugins *plugin.Plugins, control ) } -func (c *SinkController) OpenPipeline(onClose func(ctx context.Context, cause string)) (pipeline.Pipeline, error) { +func (c *SinkController) OpenPipeline(sinkKey key.SinkKey, onClose func(ctx context.Context, cause string)) (pipeline.Pipeline, error) { if c.PipelineOpenError != nil { return nil, c.PipelineOpenError } - return &Pipeline{controller: c, onClose: onClose}, nil + return &Pipeline{controller: c, sinkKey: sinkKey, onClose: onClose}, nil } func (p *Pipeline) ReopenOnSinkModification() bool { @@ -127,6 +161,13 @@ func (p *Pipeline) ReopenOnSinkModification() bool { } func (p *Pipeline) WriteRecord(_ recordctx.Context) (pipeline.WriteResult, error) { + p.controller.recordWrite(p.sinkKey) + if hook := p.controller.PipelineWriteHook; hook != nil { + if err := hook(p.sinkKey); err != nil { + return pipeline.WriteResult{Status: pipeline.RecordError}, err + } + return pipeline.WriteResult{Status: p.controller.PipelineWriteRecordStatus}, nil + } if err := p.controller.PipelineWriteError; err != nil { return pipeline.WriteResult{Status: pipeline.RecordError}, err } diff --git a/internal/pkg/service/stream/storage/test/source.go b/internal/pkg/service/stream/storage/test/source.go index c4beb73edc..54df17ee4a 100644 --- a/internal/pkg/service/stream/storage/test/source.go +++ b/internal/pkg/service/stream/storage/test/source.go @@ -27,3 +27,13 @@ func NewHTTPSource(k key.SourceKey) definition.Source { HTTP: &definition.HTTPSource{Secret: "012345678901234567890123456789012345678912345678"}, } } + +func NewOTLPSource(k key.SourceKey) definition.Source { + return definition.Source{ + SourceKey: k, + Type: definition.SourceTypeOTLP, + Name: "My OTLP Source", + Description: "My Description", + OTLP: &definition.OTLPSource{Secret: "012345678901234567890123456789012345678912345678"}, + } +} diff --git a/scripts/stream-otlp-setup.sh b/scripts/stream-otlp-setup.sh new file mode 100755 index 0000000000..c2559456d8 --- /dev/null +++ b/scripts/stream-otlp-setup.sh @@ -0,0 +1,436 @@ +#!/usr/bin/env bash +# scripts/stream-otlp-setup.sh +# +# Creates an OTLP source + three signal-specific sinks (logs, metrics, traces) +# on a Keboola Stream stack. Every field from the OTLP flatten step gets its +# own column so nothing is dropped. +# +# State is saved incrementally to ./stream-otlp-state.env (relative to the +# current working directory) after each step so a partial failure can be +# resumed by re-running the script. +# +# Usage: +# export KEBOOLA_TOKEN= +# export KEBOOLA_BRANCH_ID= # 0 = default +# bash scripts/stream-otlp-setup.sh +# +# Optional overrides: +# STREAM_API_HOST — defaults to stream.keboola.com +# SOURCE_NAME — defaults to "OTLP Source" +# BUCKET — storage bucket prefix, defaults to in.c-otlp +# CLEANUP — set to "true" to delete the source instead + +set -euo pipefail + +# Upfront dependency check — the script relies on jq for response parsing, +# state extraction, and payload construction. Fail fast instead of half-creating +# resources on a system without it. +command -v jq &>/dev/null || { + echo " ✗ jq is required but not installed. Install via your package manager (apt, brew, dnf, …)." >&2 + exit 1 +} +command -v curl &>/dev/null || { + echo " ✗ curl is required but not installed." >&2 + exit 1 +} + +TOKEN="${KEBOOLA_TOKEN:?Set KEBOOLA_TOKEN}" +BRANCH_ID="${KEBOOLA_BRANCH_ID:?Set KEBOOLA_BRANCH_ID (0 = default branch)}" +STREAM_API_HOST="${STREAM_API_HOST:-stream.keboola.com}" +SOURCE_NAME="${SOURCE_NAME:-OTLP Source}" +BUCKET="${BUCKET:-in.c-otlp}" +STATE_FILE="${STATE_FILE:-./stream-otlp-state.env}" + +STREAM_API="https://${STREAM_API_HOST}/v1" + +pretty() { jq .; } +header() { echo; echo "══════════════════════════════════════════════════════"; echo " $*"; echo "══════════════════════════════════════════════════════"; } +ok() { echo " ✓ $*"; } +info() { echo " → $*"; } +warn() { echo " ! $*"; } +fail() { echo " ✗ $*" >&2; exit 1; } + +# api_post → prints response body; sets API_CODE and API_BODY +api_post() { + local path="$1" body="$2" + local raw + raw=$(curl -s --max-time 30 -w "\n%{http_code}" -X POST \ + "${STREAM_API}${path}" \ + -H "Content-Type: application/json" \ + -H "X-StorageApi-Token: ${TOKEN}" \ + -d "${body}") + API_CODE=$(tail -1 <<< "${raw}") + API_BODY=$(head -n -1 <<< "${raw}") + echo "${API_BODY}" | pretty +} + +# api_get → sets API_CODE and API_BODY +api_get() { + local path="$1" + local raw + raw=$(curl -s --max-time 30 -w "\n%{http_code}" \ + "${STREAM_API}${path}" \ + -H "X-StorageApi-Token: ${TOKEN}") + API_CODE=$(tail -1 <<< "${raw}") + API_BODY=$(head -n -1 <<< "${raw}") +} + +poll_task() { + local task_url="$1" + local elapsed=0 response status + while true; do + response=$(curl -s --max-time 30 "${task_url}" -H "X-StorageApi-Token: ${TOKEN}") + status=$(echo "${response}" | jq -r '.status') + if [[ "${status}" != "processing" ]]; then + echo "${response}" + [[ "${status}" == "success" ]] || fail "Task failed: $(echo "${response}" | jq -r '.error // .result // "unknown"')" + return 0 + fi + elapsed=$((elapsed + 2)) + [[ ${elapsed} -lt 60 ]] || fail "Task timed out after 60s" + sleep 2 + done +} + +save_state() { + # Create with restrictive perms — OTLP_URL embeds the write secret and the + # default umask on shared machines may otherwise leave it readable to others. + ( umask 077; : > "${STATE_FILE}" ) + chmod 600 "${STATE_FILE}" + cat > "${STATE_FILE}" < /dev/null + ok "Delete task completed" + fi + rm -f "${STATE_FILE}" + ok "Done" + exit 0 +fi + +# ── 1. Create OTLP source ───────────────────────────────────────────────────── + +header "1. Create OTLP source" + +if [[ -n "${SOURCE_ID}" ]]; then + ok "Already exists: ${SOURCE_ID} — skipping" +else + # Build the payload with jq so SOURCE_NAME values containing quotes, backslashes, + # or newlines are escaped correctly. Direct string interpolation would produce + # invalid JSON for any non-trivial name. + create_payload=$(jq -nc --arg name "${SOURCE_NAME}" '{name: $name, type: "otlp"}') + api_post "/branches/${BRANCH_ID}/sources" "${create_payload}" + + if [[ "${API_CODE}" == "409" ]]; then + warn "409 — fetching existing OTLP source by name…" + api_get "/branches/${BRANCH_ID}/sources" + # Filter by both name AND type so we never accidentally reuse (and later + # delete during cleanup) an unrelated HTTP source that happens to share + # the name. + SOURCE_ID=$(echo "${API_BODY}" | jq -r --arg n "${SOURCE_NAME}" \ + '.sources[] | select(.name==$n and .type=="otlp") | .sourceId' | head -1) + if [[ -z "${SOURCE_ID}" ]]; then + # A non-OTLP source matched the name → bail out instead of reusing it. + conflicting_type=$(echo "${API_BODY}" | jq -r --arg n "${SOURCE_NAME}" \ + '[.sources[] | select(.name==$n) | .type] | first // empty') + if [[ -n "${conflicting_type}" ]]; then + fail "A source named '${SOURCE_NAME}' already exists with type '${conflicting_type}'. Pick a different SOURCE_NAME." + fi + fail "OTLP source '${SOURCE_NAME}' not found in list" + fi + ok "Reusing OTLP source: ${SOURCE_ID}" + elif [[ "${API_CODE}" -ge 400 ]]; then + fail "Create source failed HTTP ${API_CODE}" + else + TASK_URL=$(echo "${API_BODY}" | jq -r '.url') + info "Polling…" + TASK=$(poll_task "${TASK_URL}") + SOURCE_ID=$(echo "${TASK}" | jq -r '.outputs.sourceId') + ok "Created: ${SOURCE_ID}" + fi + save_state +fi + +# ── 2. Fetch OTLP URL ───────────────────────────────────────────────────────── + +header "2. Get OTLP ingestion URL" + +if [[ -n "${OTLP_URL}" ]]; then + ok "Already have: ${OTLP_URL}" +else + api_get "/branches/${BRANCH_ID}/sources/${SOURCE_ID}" + OTLP_URL=$(echo "${API_BODY}" | jq -r '.otlp.url') + [[ -n "${OTLP_URL}" && "${OTLP_URL}" != "null" ]] || fail "Missing .otlp.url" + ok "OTLP URL: ${OTLP_URL}" + save_state +fi + +# ── Sink helper ─────────────────────────────────────────────────────────────── + +create_sink() { + local sink_var="$1" sink_name="$2" table_id="$3" columns_json="$4" signals_json="$5" + + header "Create sink: ${sink_name} → ${table_id}" + + local current_id="${!sink_var}" + if [[ -n "${current_id}" ]]; then + ok "Already exists: ${current_id} — skipping" + return + fi + + local body + body=$(jq -nc \ + --arg name "${sink_name}" \ + --arg tableId "${table_id}" \ + --argjson columns "${columns_json}" \ + --argjson signals "${signals_json}" \ + '{name: $name, type: "table", allowedSignals: $signals, table: {type: "keboola", tableId: $tableId, mapping: {columns: $columns}}}') + + api_post "/branches/${BRANCH_ID}/sources/${SOURCE_ID}/sinks" "${body}" + + local sink_id + if [[ "${API_CODE}" == "409" ]]; then + warn "409 — fetching existing sink…" + api_get "/branches/${BRANCH_ID}/sources/${SOURCE_ID}/sinks" + # An existing sink with the same name might point at a different table + # or carry a different allowedSignals filter. Verify both before reusing + # so a stale fixture doesn't silently route to the wrong table/signal. + matched=$(echo "${API_BODY}" | jq -c --arg n "${sink_name}" \ + '[.sinks[] | select(.name==$n)] | first // empty') + if [[ -z "${matched}" ]]; then + fail "Sink '${sink_name}' not found in list" + fi + sink_id=$(jq -r '.sinkId' <<< "${matched}") + existing_table=$(jq -r '.table.tableId // empty' <<< "${matched}") + existing_signals=$(jq -c '.allowedSignals // []' <<< "${matched}") + expected_signals=$(jq -c '.' <<< "${signals_json}") + if [[ "${existing_table}" != "${table_id}" ]]; then + fail "Sink '${sink_name}' (${sink_id}) targets table '${existing_table}', expected '${table_id}'. Refusing to reuse." + fi + if [[ "${existing_signals}" != "${expected_signals}" ]]; then + fail "Sink '${sink_name}' (${sink_id}) has allowedSignals=${existing_signals}, expected ${expected_signals}. Refusing to reuse." + fi + ok "Reusing sink with matching mapping: ${sink_id}" + elif [[ "${API_CODE}" -ge 400 ]]; then + fail "Create sink failed HTTP ${API_CODE}" + else + local task_url + task_url=$(echo "${API_BODY}" | jq -r '.url') + info "Polling…" + local task + task=$(poll_task "${task_url}") + sink_id=$(echo "${task}" | jq -r '.outputs.sinkId') + ok "Created: ${sink_id}" + fi + + # Write back to the named variable and persist + printf -v "${sink_var}" '%s' "${sink_id}" + save_state +} + +# ── 3. Logs sink ────────────────────────────────────────────────────────────── +# One row per log record. Every field from flatten_logs.go gets its own column. + +LOGS_COLUMNS=$(cat <<'EOF' +[ + {"type":"datetime","name":"datetime"}, + {"type":"template","name":"signal", "template":{"language":"jsonnet","content":"\"logs\""}}, + {"type":"template","name":"timestamp", "template":{"language":"jsonnet","content":"Body(\"timestamp\", \"\")"}}, + {"type":"template","name":"observed_timestamp", "template":{"language":"jsonnet","content":"Body(\"observed_timestamp\", \"\")"}}, + {"type":"template","name":"severity_number", "template":{"language":"jsonnet","content":"Body(\"severity_number\", \"\")"}}, + {"type":"template","name":"severity_text", "template":{"language":"jsonnet","content":"Body(\"severity_text\", \"\")"}}, + {"type":"template","name":"body", "template":{"language":"jsonnet","content":"Body(\"body\", \"\")"}}, + {"type":"template","name":"flags", "template":{"language":"jsonnet","content":"Body(\"flags\", \"\")"}}, + {"type":"template","name":"trace_id", "template":{"language":"jsonnet","content":"Body(\"trace_id\", \"\")"}}, + {"type":"template","name":"span_id", "template":{"language":"jsonnet","content":"Body(\"span_id\", \"\")"}}, + {"type":"template","name":"attributes", "template":{"language":"jsonnet","content":"Body(\"attributes\", {})"}}, + {"type":"template","name":"resource", "template":{"language":"jsonnet","content":"Body(\"resource\", {})"}}, + {"type":"template","name":"scope_name", "template":{"language":"jsonnet","content":"Body(\"scope\", {})[\"name\"]"}}, + {"type":"template","name":"scope_version", "template":{"language":"jsonnet","content":"Body(\"scope\", {})[\"version\"]"}} +] +EOF +) + +create_sink "SINK_ID_LOGS" "OTLP Logs" "${BUCKET}.logs" "${LOGS_COLUMNS}" '["logs"]' + +# ── 4. Metrics sink ─────────────────────────────────────────────────────────── +# One row per data point. Covers gauge, sum, histogram, exp-histogram, summary. + +METRICS_COLUMNS=$(cat <<'EOF' +[ + {"type":"datetime","name":"datetime"}, + {"type":"template","name":"signal", "template":{"language":"jsonnet","content":"\"metrics\""}}, + {"type":"template","name":"timestamp", "template":{"language":"jsonnet","content":"Body(\"timestamp\", \"\")"}}, + {"type":"template","name":"start_timestamp", "template":{"language":"jsonnet","content":"Body(\"start_timestamp\", \"\")"}}, + {"type":"template","name":"metric_name", "template":{"language":"jsonnet","content":"Body(\"metric_name\", \"\")"}}, + {"type":"template","name":"metric_description", "template":{"language":"jsonnet","content":"Body(\"metric_description\", \"\")"}}, + {"type":"template","name":"metric_unit", "template":{"language":"jsonnet","content":"Body(\"metric_unit\", \"\")"}}, + {"type":"template","name":"metric_type", "template":{"language":"jsonnet","content":"Body(\"metric_type\", \"\")"}}, + {"type":"template","name":"value", "template":{"language":"jsonnet","content":"Body(\"value\", \"\")"}}, + {"type":"template","name":"count", "template":{"language":"jsonnet","content":"Body(\"count\", \"\")"}}, + {"type":"template","name":"sum", "template":{"language":"jsonnet","content":"Body(\"sum\", \"\")"}}, + {"type":"template","name":"min", "template":{"language":"jsonnet","content":"Body(\"min\", \"\")"}}, + {"type":"template","name":"max", "template":{"language":"jsonnet","content":"Body(\"max\", \"\")"}}, + {"type":"template","name":"bucket_counts", "template":{"language":"jsonnet","content":"Body(\"bucket_counts\", [])"}}, + {"type":"template","name":"explicit_bounds", "template":{"language":"jsonnet","content":"Body(\"explicit_bounds\", [])"}}, + {"type":"template","name":"is_monotonic", "template":{"language":"jsonnet","content":"Body(\"is_monotonic\", \"\")"}}, + {"type":"template","name":"aggregation_temporality", "template":{"language":"jsonnet","content":"Body(\"aggregation_temporality\", \"\")"}}, + {"type":"template","name":"scale", "template":{"language":"jsonnet","content":"Body(\"scale\", \"\")"}}, + {"type":"template","name":"zero_count", "template":{"language":"jsonnet","content":"Body(\"zero_count\", \"\")"}}, + {"type":"template","name":"quantile_values", "template":{"language":"jsonnet","content":"Body(\"quantile_values\", [])"}}, + {"type":"template","name":"attributes", "template":{"language":"jsonnet","content":"Body(\"attributes\", {})"}}, + {"type":"template","name":"resource", "template":{"language":"jsonnet","content":"Body(\"resource\", {})"}}, + {"type":"template","name":"scope_name", "template":{"language":"jsonnet","content":"Body(\"scope\", {})[\"name\"]"}}, + {"type":"template","name":"scope_version", "template":{"language":"jsonnet","content":"Body(\"scope\", {})[\"version\"]"}} +] +EOF +) + +create_sink "SINK_ID_METRICS" "OTLP Metrics" "${BUCKET}.metrics" "${METRICS_COLUMNS}" '["metrics"]' + +# ── 5. Traces sink ──────────────────────────────────────────────────────────── +# One row per span. Events and links stored as JSON arrays. + +TRACES_COLUMNS=$(cat <<'EOF' +[ + {"type":"datetime","name":"datetime"}, + {"type":"template","name":"signal", "template":{"language":"jsonnet","content":"\"traces\""}}, + {"type":"template","name":"timestamp", "template":{"language":"jsonnet","content":"Body(\"timestamp\", \"\")"}}, + {"type":"template","name":"end_timestamp", "template":{"language":"jsonnet","content":"Body(\"end_timestamp\", \"\")"}}, + {"type":"template","name":"trace_id", "template":{"language":"jsonnet","content":"Body(\"trace_id\", \"\")"}}, + {"type":"template","name":"span_id", "template":{"language":"jsonnet","content":"Body(\"span_id\", \"\")"}}, + {"type":"template","name":"parent_span_id", "template":{"language":"jsonnet","content":"Body(\"parent_span_id\", \"\")"}}, + {"type":"template","name":"trace_state", "template":{"language":"jsonnet","content":"Body(\"trace_state\", \"\")"}}, + {"type":"template","name":"name", "template":{"language":"jsonnet","content":"Body(\"name\", \"\")"}}, + {"type":"template","name":"kind", "template":{"language":"jsonnet","content":"Body(\"kind\", \"\")"}}, + {"type":"template","name":"flags", "template":{"language":"jsonnet","content":"Body(\"flags\", \"\")"}}, + {"type":"template","name":"status_code", "template":{"language":"jsonnet","content":"Body(\"status_code\", \"\")"}}, + {"type":"template","name":"status_message", "template":{"language":"jsonnet","content":"Body(\"status_message\", \"\")"}}, + {"type":"template","name":"attributes", "template":{"language":"jsonnet","content":"Body(\"attributes\", {})"}}, + {"type":"template","name":"events", "template":{"language":"jsonnet","content":"Body(\"events\", [])"}}, + {"type":"template","name":"links", "template":{"language":"jsonnet","content":"Body(\"links\", [])"}}, + {"type":"template","name":"resource", "template":{"language":"jsonnet","content":"Body(\"resource\", {})"}}, + {"type":"template","name":"scope_name", "template":{"language":"jsonnet","content":"Body(\"scope\", {})[\"name\"]"}}, + {"type":"template","name":"scope_version", "template":{"language":"jsonnet","content":"Body(\"scope\", {})[\"version\"]"}} +] +EOF +) + +create_sink "SINK_ID_TRACES" "OTLP Traces" "${BUCKET}.traces" "${TRACES_COLUMNS}" '["traces"]' + +# ── Done ────────────────────────────────────────────────────────────────────── + +header "Done — OTLP pipeline ready" +echo +echo " Source: ${SOURCE_ID}" +echo " Sink (logs): ${SINK_ID_LOGS} → ${BUCKET}.logs" +echo " Sink (metrics): ${SINK_ID_METRICS} → ${BUCKET}.metrics" +echo " Sink (traces): ${SINK_ID_TRACES} → ${BUCKET}.traces" +echo +echo "══════════════════════════════════════════════════════" +echo " Step 1 — Smoke-test the endpoint" +echo "══════════════════════════════════════════════════════" +echo +echo " Send a minimal logs batch with curl (protobuf payload omitted for brevity —" +echo " any OTel SDK will produce valid encodings):" +echo +echo " curl -i -X POST \"${OTLP_URL}/v1/logs\" \\" +echo " -H 'Content-Type: application/json' \\" +echo " -d '{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{\"body\":{\"stringValue\":\"hello\"}}]}]}]}'" +echo +echo " Expect HTTP 200 with an empty/partial-success body." +echo +echo "══════════════════════════════════════════════════════" +echo " Step 2 — Configure your OTel SDK" +echo "══════════════════════════════════════════════════════" +echo +echo " Set these two environment variables in every service:" +echo +echo " OTEL_EXPORTER_OTLP_ENDPOINT=${OTLP_URL}" +echo " OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" +echo +echo " The secret is embedded in the URL — no extra auth header needed." +echo +echo " Language quick-starts:" +echo +echo " Go (using OTLP env vars — works with any OTel Go SDK exporter):" +echo " import _ \"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp\"" +echo " // Set env vars above, then use sdk.WithEnv() on the provider." +echo +echo " Python:" +echo " pip install opentelemetry-exporter-otlp" +echo " # Env vars are picked up automatically by OTLPLogExporter(), etc." +echo +echo " Node.js:" +echo " npm install @opentelemetry/exporter-logs-otlp-http" +echo " // Env vars are read by @opentelemetry/exporter-*-otlp-http packages." +echo +echo " Java / Spring Boot:" +echo " management.otlp.logging.endpoint=\${OTEL_EXPORTER_OTLP_ENDPOINT}/v1/logs" +echo " # or set OTEL_EXPORTER_OTLP_ENDPOINT as a JVM system property." +echo +echo "══════════════════════════════════════════════════════" +echo " Step 3 — Verify rows in Keboola Storage" +echo "══════════════════════════════════════════════════════" +echo +echo " Wait ~30 s after the first payload, then run in Keboola Transformations:" +echo +echo " Snowflake:" +echo " SELECT datetime, timestamp, severity_text, body," +echo " attributes:\"user.id\"::string AS user_id" +echo " FROM \"${BUCKET}\".\"logs\"" +echo " ORDER BY datetime DESC LIMIT 20;" +echo +echo " BigQuery:" +echo " SELECT datetime, timestamp, severity_text, body," +echo " JSON_VALUE(attributes, '$.user.id') AS user_id" +echo " FROM \`${BUCKET//./_}.logs\`" +echo " ORDER BY datetime DESC LIMIT 20;" +echo +echo "══════════════════════════════════════════════════════" +echo " Tear down" +echo "══════════════════════════════════════════════════════" +echo +echo " CLEANUP=true bash scripts/stream-otlp-setup.sh" diff --git a/test/stream/api/source/create-005-invalid-type-400/001-create-source/expected-response.json b/test/stream/api/source/create-005-invalid-type-400/001-create-source/expected-response.json index 158d5ac8be..8bd543dc9d 100644 --- a/test/stream/api/source/create-005-invalid-type-400/001-create-source/expected-response.json +++ b/test/stream/api/source/create-005-invalid-type-400/001-create-source/expected-response.json @@ -1,5 +1,5 @@ { "statusCode": 400, "error": "stream.api.invalidEnumValue", - "message": "Value of body.type must be one of \"http\" but got value \"foo\"." + "message": "Value of body.type must be one of \"http\", \"otlp\" but got value \"foo\"." } diff --git a/test/stream/api/source/create-otlp-source/001-create-source/expected-http-code b/test/stream/api/source/create-otlp-source/001-create-source/expected-http-code new file mode 100644 index 0000000000..252b382b33 --- /dev/null +++ b/test/stream/api/source/create-otlp-source/001-create-source/expected-http-code @@ -0,0 +1 @@ +202 \ No newline at end of file diff --git a/test/stream/api/source/create-otlp-source/001-create-source/expected-response.json b/test/stream/api/source/create-otlp-source/001-create-source/expected-response.json new file mode 100644 index 0000000000..aab978af40 --- /dev/null +++ b/test/stream/api/source/create-otlp-source/001-create-source/expected-response.json @@ -0,0 +1,8 @@ +{ + "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "type": "api.create.source", + "url": "https://stream.keboola.local/v1/tasks/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "status": "processing", + "isFinished": false, + "createdAt": "%s" +} diff --git a/test/stream/api/source/create-otlp-source/001-create-source/request.json b/test/stream/api/source/create-otlp-source/001-create-source/request.json new file mode 100644 index 0000000000..3156507cdf --- /dev/null +++ b/test/stream/api/source/create-otlp-source/001-create-source/request.json @@ -0,0 +1,12 @@ +{ + "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "body": { + "name": "My OTLP Source", + "type": "otlp" + } +} diff --git a/test/stream/api/source/create-otlp-source/002-poll-source/expected-http-code b/test/stream/api/source/create-otlp-source/002-poll-source/expected-http-code new file mode 100644 index 0000000000..ae4ee13c08 --- /dev/null +++ b/test/stream/api/source/create-otlp-source/002-poll-source/expected-http-code @@ -0,0 +1 @@ +200 \ No newline at end of file diff --git a/test/stream/api/source/create-otlp-source/002-poll-source/expected-response.json b/test/stream/api/source/create-otlp-source/002-poll-source/expected-response.json new file mode 100644 index 0000000000..2384f0ee9e --- /dev/null +++ b/test/stream/api/source/create-otlp-source/002-poll-source/expected-response.json @@ -0,0 +1,17 @@ +{ + "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "type": "api.create.source", + "url": "https://stream.keboola.local/v1/tasks/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "status": "success", + "isFinished": true, + "createdAt": "%s", + "finishedAt": "%s", + "duration": %d, + "result": "Source has been created successfully.", + "outputs": { + "url": "https://stream.keboola.local/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source", + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source" + } +} diff --git a/test/stream/api/source/create-otlp-source/002-poll-source/request.json b/test/stream/api/source/create-otlp-source/002-poll-source/request.json new file mode 100644 index 0000000000..59eb032876 --- /dev/null +++ b/test/stream/api/source/create-otlp-source/002-poll-source/request.json @@ -0,0 +1,11 @@ +{ + "path": "<<001-create-source:response.url>>", + "method": "GET", + "headers": { + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "repeat": { + "until": "status != 'processing'", + "timeout": 60 + } +} diff --git a/test/stream/api/source/create-otlp-source/003-get-source/expected-http-code b/test/stream/api/source/create-otlp-source/003-get-source/expected-http-code new file mode 100644 index 0000000000..ae4ee13c08 --- /dev/null +++ b/test/stream/api/source/create-otlp-source/003-get-source/expected-http-code @@ -0,0 +1 @@ +200 \ No newline at end of file diff --git a/test/stream/api/source/create-otlp-source/003-get-source/expected-response.json b/test/stream/api/source/create-otlp-source/003-get-source/expected-response.json new file mode 100644 index 0000000000..918df3f2a2 --- /dev/null +++ b/test/stream/api/source/create-otlp-source/003-get-source/expected-response.json @@ -0,0 +1,36 @@ +{ + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source", + "type": "otlp", + "name": "My OTLP Source", + "description": "", + "otlp": { + "url": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-otlp-source/%s", + "baseUrl": "https://stream-in.keboola.local/otlp/%%TEST_KBC_PROJECT_ID%%/my-otlp-source", + "secret": "%s" + }, + "version": { + "number": 1, + "hash": "%s", + "description": "New source.", + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + }, + "created": { + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + } +} diff --git a/test/stream/api/source/create-otlp-source/003-get-source/request.json b/test/stream/api/source/create-otlp-source/003-get-source/request.json new file mode 100644 index 0000000000..31fb6bb97f --- /dev/null +++ b/test/stream/api/source/create-otlp-source/003-get-source/request.json @@ -0,0 +1,7 @@ +{ + "path": "<<002-poll-source:response.outputs.url>>", + "method": "GET", + "headers": { + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + } +} diff --git a/test/stream/api/source/create-otlp-source/expected-etcd-kvs.txt b/test/stream/api/source/create-otlp-source/expected-etcd-kvs.txt new file mode 100644 index 0000000000..83bea1e97e --- /dev/null +++ b/test/stream/api/source/create-otlp-source/expected-etcd-kvs.txt @@ -0,0 +1,117 @@ +<<<<< +definition/branch/active/%%TEST_KBC_PROJECT_ID%%/%%TEST_DEFAULT_BRANCH_ID%% +----- +{ + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "created": { + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + }, + "isDefault": true +} +>>>>> + +<<<<< +definition/source/active/%%TEST_KBC_PROJECT_ID%%/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source +----- +{ + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source", + "created": { + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + }, + "version": { + "number": 1, + "hash": "%s", + "description": "New source.", + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + }, + "type": "otlp", + "name": "My OTLP Source", + "otlp": { + "secret": "%s" + } +} +>>>>> + +<<<<< +definition/source/version/%%TEST_KBC_PROJECT_ID%%/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/0000000001 +----- +{ + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source", + "created": { + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + }, + "version": { + "number": 1, + "hash": "%s", + "description": "New source.", + "at": "%s", + "by": { + "type": "user", + "tokenId": "%s", + "tokenDesc": "%s", + "userId": "%s", + "userName": "%s" + } + }, + "type": "otlp", + "name": "My OTLP Source", + "otlp": { + "secret": "%s" + } +} +>>>>> + +<<<<< +task/%%TEST_KBC_PROJECT_ID%%/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s +----- +{ + "projectId": %%TEST_KBC_PROJECT_ID%%, + "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "type": "api.create.source", + "createdAt": "%s", + "finishedAt": "%s", + "node": "test-node", + "lock": "runtime/lock/task/%%TEST_KBC_PROJECT_ID%%/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source", + "result": "Source has been created successfully.", + "outputs": { + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "projectId": %%TEST_KBC_PROJECT_ID%%, + "sourceId": "my-otlp-source", + "url": "https://stream.keboola.local/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source" + }, + "duration": %s +} +>>>>> diff --git a/test/stream/api/source/otlp-test-mapping/001-create-source/expected-http-code b/test/stream/api/source/otlp-test-mapping/001-create-source/expected-http-code new file mode 100644 index 0000000000..252b382b33 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/001-create-source/expected-http-code @@ -0,0 +1 @@ +202 \ No newline at end of file diff --git a/test/stream/api/source/otlp-test-mapping/001-create-source/expected-response.json b/test/stream/api/source/otlp-test-mapping/001-create-source/expected-response.json new file mode 100644 index 0000000000..aab978af40 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/001-create-source/expected-response.json @@ -0,0 +1,8 @@ +{ + "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "type": "api.create.source", + "url": "https://stream.keboola.local/v1/tasks/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "status": "processing", + "isFinished": false, + "createdAt": "%s" +} diff --git a/test/stream/api/source/otlp-test-mapping/001-create-source/request.json b/test/stream/api/source/otlp-test-mapping/001-create-source/request.json new file mode 100644 index 0000000000..3156507cdf --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/001-create-source/request.json @@ -0,0 +1,12 @@ +{ + "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "body": { + "name": "My OTLP Source", + "type": "otlp" + } +} diff --git a/test/stream/api/source/otlp-test-mapping/002-poll-source/expected-http-code b/test/stream/api/source/otlp-test-mapping/002-poll-source/expected-http-code new file mode 100644 index 0000000000..ae4ee13c08 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/002-poll-source/expected-http-code @@ -0,0 +1 @@ +200 \ No newline at end of file diff --git a/test/stream/api/source/otlp-test-mapping/002-poll-source/expected-response.json b/test/stream/api/source/otlp-test-mapping/002-poll-source/expected-response.json new file mode 100644 index 0000000000..2384f0ee9e --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/002-poll-source/expected-response.json @@ -0,0 +1,17 @@ +{ + "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "type": "api.create.source", + "url": "https://stream.keboola.local/v1/tasks/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/%s", + "status": "success", + "isFinished": true, + "createdAt": "%s", + "finishedAt": "%s", + "duration": %d, + "result": "Source has been created successfully.", + "outputs": { + "url": "https://stream.keboola.local/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source", + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source" + } +} diff --git a/test/stream/api/source/otlp-test-mapping/002-poll-source/request.json b/test/stream/api/source/otlp-test-mapping/002-poll-source/request.json new file mode 100644 index 0000000000..59eb032876 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/002-poll-source/request.json @@ -0,0 +1,11 @@ +{ + "path": "<<001-create-source:response.url>>", + "method": "GET", + "headers": { + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "repeat": { + "until": "status != 'processing'", + "timeout": 60 + } +} diff --git a/test/stream/api/source/otlp-test-mapping/003-create-sink/expected-http-code b/test/stream/api/source/otlp-test-mapping/003-create-sink/expected-http-code new file mode 100644 index 0000000000..252b382b33 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/003-create-sink/expected-http-code @@ -0,0 +1 @@ +202 \ No newline at end of file diff --git a/test/stream/api/source/otlp-test-mapping/003-create-sink/expected-response.json b/test/stream/api/source/otlp-test-mapping/003-create-sink/expected-response.json new file mode 100644 index 0000000000..a960ed6067 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/003-create-sink/expected-response.json @@ -0,0 +1,8 @@ +{ + "taskId": "api.create.sink/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/my-sink/%s", + "type": "api.create.sink", + "url": "https://stream.keboola.local/v1/tasks/api.create.sink/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/my-sink/%s", + "status": "processing", + "isFinished": false, + "createdAt": "%s" +} diff --git a/test/stream/api/source/otlp-test-mapping/003-create-sink/request.json b/test/stream/api/source/otlp-test-mapping/003-create-sink/request.json new file mode 100644 index 0000000000..6a825de48b --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/003-create-sink/request.json @@ -0,0 +1,64 @@ +{ + "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source/sinks", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "body": { + "name": "My Sink", + "type": "table", + "table": { + "type": "keboola", + "tableId": "in.c-otlp.logs", + "mapping": { + "columns": [ + { + "type": "datetime", + "name": "datetime" + }, + { + "type": "template", + "name": "severity", + "template": { + "language": "jsonnet", + "content": "Body('severity_text')" + } + }, + { + "type": "template", + "name": "message", + "template": { + "language": "jsonnet", + "content": "Body('body')" + } + }, + { + "type": "template", + "name": "service", + "template": { + "language": "jsonnet", + "content": "Body('resource')['service.name']" + } + }, + { + "type": "template", + "name": "otlp_timestamp", + "template": { + "language": "jsonnet", + "content": "Body('timestamp')" + } + }, + { + "type": "template", + "name": "user_id", + "template": { + "language": "jsonnet", + "content": "Body('attributes')['user.id']" + } + } + ] + } + } + } +} diff --git a/test/stream/api/source/otlp-test-mapping/004-poll-sink/expected-http-code b/test/stream/api/source/otlp-test-mapping/004-poll-sink/expected-http-code new file mode 100644 index 0000000000..ae4ee13c08 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/004-poll-sink/expected-http-code @@ -0,0 +1 @@ +200 \ No newline at end of file diff --git a/test/stream/api/source/otlp-test-mapping/004-poll-sink/expected-response.json b/test/stream/api/source/otlp-test-mapping/004-poll-sink/expected-response.json new file mode 100644 index 0000000000..b076de5c05 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/004-poll-sink/expected-response.json @@ -0,0 +1,18 @@ +{ + "taskId": "api.create.sink/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/my-sink/%s", + "type": "api.create.sink", + "url": "https://stream.keboola.local/v1/tasks/api.create.sink/%%TEST_DEFAULT_BRANCH_ID%%/my-otlp-source/my-sink/%s", + "status": "success", + "isFinished": true, + "createdAt": "%s", + "finishedAt": "%s", + "duration": %d, + "result": "Sink has been created successfully.", + "outputs": { + "url": "https://stream.keboola.local/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source/sinks/my-sink", + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source", + "sinkId": "my-sink" + } +} diff --git a/test/stream/api/source/otlp-test-mapping/004-poll-sink/request.json b/test/stream/api/source/otlp-test-mapping/004-poll-sink/request.json new file mode 100644 index 0000000000..13f03bc884 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/004-poll-sink/request.json @@ -0,0 +1,11 @@ +{ + "path": "<<003-create-sink:response.url>>", + "method": "GET", + "headers": { + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "repeat": { + "until": "status != 'processing'", + "timeout": 60 + } +} diff --git a/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/expected-http-code b/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/expected-http-code new file mode 100644 index 0000000000..ae4ee13c08 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/expected-http-code @@ -0,0 +1 @@ +200 \ No newline at end of file diff --git a/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/expected-response.json b/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/expected-response.json new file mode 100644 index 0000000000..b233f1ad28 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/expected-response.json @@ -0,0 +1,41 @@ +{ + "projectId": %%TEST_KBC_PROJECT_ID%%, + "branchId": %%TEST_DEFAULT_BRANCH_ID%%, + "sourceId": "my-otlp-source", + "tables": [ + { + "sinkId": "my-sink", + "tableId": "in.c-otlp.logs", + "rows": [ + { + "columns": [ + { + "name": "datetime", + "value": "%s" + }, + { + "name": "severity", + "value": "\"WARN\"" + }, + { + "name": "message", + "value": "\"disk space low\"" + }, + { + "name": "service", + "value": "\"storage-service\"" + }, + { + "name": "otlp_timestamp", + "value": "\"2024-06-01T12:00:00Z\"" + }, + { + "name": "user_id", + "value": "\"u-42\"" + } + ] + } + ] + } + ] +} diff --git a/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/request.json b/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/request.json new file mode 100644 index 0000000000..1b8dc73562 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/005-test-mapping-logs/request.json @@ -0,0 +1,15 @@ +{ + "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source/test", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "body": { + "severity_text": "WARN", + "body": "disk space low", + "timestamp": "2024-06-01T12:00:00Z", + "resource": {"service.name": "storage-service"}, + "attributes": {"user.id": "u-42"} + } +} diff --git a/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/expected-http-code b/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/expected-http-code new file mode 100644 index 0000000000..97964b3fde --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/expected-http-code @@ -0,0 +1 @@ +422 \ No newline at end of file diff --git a/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/expected-response.json b/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/expected-response.json new file mode 100644 index 0000000000..62655e6403 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/expected-response.json @@ -0,0 +1,5 @@ +{ + "statusCode": 422, + "error": "stream.api.unprocessableContent", + "message": "Invalid value for column \"service\": path \"resource\" not found in the body." +} diff --git a/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/request.json b/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/request.json new file mode 100644 index 0000000000..cb3fb90024 --- /dev/null +++ b/test/stream/api/source/otlp-test-mapping/006-test-mapping-resource/request.json @@ -0,0 +1,14 @@ +{ + "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-otlp-source/test", + "method": "POST", + "headers": { + "Content-Type": "application/json", + "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%" + }, + "body": { + "severity_text": "ERROR", + "body": "disk full", + "timestamp": "2024-06-01T12:01:00Z", + "attributes": {"user.id": "u-99"} + } +}