feat: add trace support

Add trace search and full trace rendering for OpenTelemetry trace indexes.

This includes Grafana trace frames, service node graph frames, span events,
exception stack traces, status/warning handling, service tags, peer service
metadata, and stable per-service node colors.

Add trace-to-logs and log-to-trace correlation links for separate Quickwit
logs and traces indexes, plus configuration fields for related datasources.

Split trace parsing into dedicated backend files, add parser and frontend
link tests, document the feature, and add a local multi-service trace/log
fixture script for manual testing.

Signed-off-by: Patrik Cyvoct <patrik@ptrk.io>

Author: Sh4d1

README.md

@@ -121,6 +121,7 @@ To configure the Quickwit datasource, you need to provide the following informat
 - The index name.
 - The log message field name (optional). This is the field displayed in the explorer view.
 - The log level field name (optional). It must be a fast field.
+- The related logs or traces datasource (optional). This enables trace-to-logs and log-to-trace links when logs and traces are stored in separate Quickwit indexes.
 
 ### With Grafana UI
 
@@ -141,6 +142,112 @@ datasources:
       logLevelField: severity_text
 ```
 
+### Logs and traces in separate indexes
+
+When logs and traces are stored in different Quickwit indexes, configure one datasource per index and link them with `logsDatasourceUid` and `tracesDatasourceUid`.
+
+```yaml
+apiVersion: 1
+
+datasources:
+  - name: Quickwit Logs
+    uid: quickwit-logs
+    type: quickwit-quickwit-datasource
+    url: http://localhost:7280/api/v1
+    jsonData:
+      index: 'otel-logs-v0_9'
+      logMessageField: body.message
+      logLevelField: severity_text
+      tracesDatasourceUid: quickwit-traces
+      tracesDatasourceName: Quickwit Traces
+
+  - name: Quickwit Traces
+    uid: quickwit-traces
+    type: quickwit-quickwit-datasource
+    url: http://localhost:7280/api/v1
+    jsonData:
+      index: 'otel-traces-v0_9'
+      logsDatasourceUid: quickwit-logs
+      logsDatasourceName: Quickwit Logs
+```
+
+## Traces
+
+The query editor has two trace query types:
+
+- **Trace search** scans matching spans and returns one row per trace. Use this to find trace IDs by Lucene query, service, operation, status, or attributes.
+- **Traces** returns a full trace frame for Grafana's trace viewer. Use this with a trace ID query such as `trace_id:abc123`.
+
+The trace parser expects Quickwit OpenTelemetry trace fields such as:
+
+- `trace_id`
+- `span_id`
+- `parent_span_id`
+- `service_name`
+- `span_name`
+- `span_start_timestamp_nanos`
+- `span_duration_millis` or `span_end_timestamp_nanos`
+
+It also reads optional fields for richer trace rendering:
+
+- `resource_attributes` for service tags
+- `span_attributes` for span tags, including `service.peer.name` and `peer.service`
+- `span_status` for error status and warnings
+- `events` for span events and exception stack traces
+- `links` for span references
+- `scope_name` and `scope_version` for instrumentation library details
+
+Trace responses include:
+
+- Grafana trace frames for the trace viewer.
+- Node graph frames that summarize service-to-service calls.
+- Span warnings for error status and dropped attributes/events/links.
+- Span event details and exception stack traces.
+- Stable per-service node colors in the node graph.
+
+### Trace/log correlations
+
+Trace-to-logs links are attached to each trace span. They query the configured logs datasource with:
+
+```text
+trace_id:${__span.traceId} AND span_id:${__span.spanId}
+```
+
+Log-to-trace links are attached to log fields named:
+
+- `trace_id`
+- `traceID`
+- `traceId`
+- `attributes.trace_id`
+
+They open the configured traces datasource with:
+
+```text
+trace_id:${__value.raw}
+```
+
+### Local trace fixtures
+
+For local testing, use the fixture script:
+
+```bash
+QUICKWIT_URL=http://127.0.0.1:7280/api/v1 ./scripts/ingest-multi-service-traces.sh
+```
+
+It writes two multi-service traces into `otel-traces-v0_9` and matching correlated logs into `otel-logs-v0_9`.
+
+In **Trace search**, filter the fixture data with:
+
+```text
+span_attributes.fixture:multi-service-trace
+```
+
+In **Traces**, open a returned trace ID with:
+
+```text
+trace_id:<trace id>
+```
+
 ## Features
 
 - Explore view.
@@ -149,6 +256,9 @@ datasources:
 - Adhoc filters.
 - Annotations
 - Explore Log Context.
+- Trace search and trace view.
+- Trace-to-logs and log-to-trace links.
+- Service node graph for trace results.
 - [Alerting](https://grafana.com/docs/grafana/latest/alerting/).
 
 ## FAQ and Limitations

pkg/quickwit/client/client.go

@@ -22,6 +22,12 @@ type ReadyStatus struct {
 
 type DatasourceInfo struct {
 	ID                         int64
+	UID                        string
+	Name                       string
+	LogsDatasourceUID          string
+	LogsDatasourceName         string
+	TracesDatasourceUID        string
+	TracesDatasourceName       string
 	HTTPClient                 *http.Client
 	URL                        string
 	Database                   string

pkg/quickwit/data_query.go

@@ -30,6 +30,10 @@ func buildMSR(queries []*Query, defaultTimeField string) ([]*es.SearchRequest, e
 
 		if isLogsQuery(q) {
 			processLogsQuery(q, b, q.RangeFrom, q.RangeTo, defaultTimeField)
+		} else if isTraceSearchQuery(q) {
+			processTraceSearchQuery(q, b, defaultTimeField)
+		} else if isTracesQuery(q) {
+			processTracesQuery(q, b, defaultTimeField)
 		} else if isDocumentQuery(q) {
 			processDocumentQuery(q, b, q.RangeFrom, q.RangeTo, defaultTimeField)
 		} else {
@@ -269,8 +273,8 @@ func getPipelineAggField(m *MetricAgg) string {
 
 func isQueryWithError(query *Query) error {
 	if len(query.BucketAggs) == 0 {
-		// If no aggregations, only document and logs queries are valid
-		if len(query.Metrics) == 0 || !(isLogsQuery(query) || isDocumentQuery(query)) {
+		// If no aggregations, only document, logs, and trace queries are valid
+		if len(query.Metrics) == 0 || !(isLogsQuery(query) || isTraceSearchQuery(query) || isTracesQuery(query) || isDocumentQuery(query)) {
 			return fmt.Errorf("invalid query, missing metrics and aggregations")
 		}
 	} else {
@@ -302,19 +306,34 @@ func isQueryWithError(query *Query) error {
 }
 
 func isLogsQuery(query *Query) bool {
-	return query.Metrics[0].Type == logsType
+	return queryMetricType(query) == logsType
+}
+
+func isTracesQuery(query *Query) bool {
+	return queryMetricType(query) == tracesType
+}
+
+func isTraceSearchQuery(query *Query) bool {
+	return queryMetricType(query) == traceSearchType
 }
 
 func isDocumentQuery(query *Query) bool {
 	return isRawDataQuery(query) || isRawDocumentQuery(query)
 }
 
 func isRawDataQuery(query *Query) bool {
-	return query.Metrics[0].Type == rawDataType
+	return queryMetricType(query) == rawDataType
 }
 
 func isRawDocumentQuery(query *Query) bool {
-	return query.Metrics[0].Type == rawDocumentType
+	return queryMetricType(query) == rawDocumentType
+}
+
+func queryMetricType(query *Query) string {
+	if query == nil || len(query.Metrics) == 0 {
+		return ""
+	}
+	return query.Metrics[0].Type
 }
 
 func processLogsQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defaultTimeField string) {
@@ -337,6 +356,18 @@ func processLogsQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defa
 	}
 }
 
+func processTracesQuery(q *Query, b *es.SearchRequestBuilder, defaultTimeField string) {
+	metric := q.Metrics[0]
+	b.Sort(es.SortOrderAsc, defaultTimeField, "epoch_nanos_int")
+	b.Size(stringToIntWithDefaultValue(metric.Settings.Get("limit").MustString(), defaultSize))
+}
+
+func processTraceSearchQuery(q *Query, b *es.SearchRequestBuilder, defaultTimeField string) {
+	metric := q.Metrics[0]
+	b.Sort(es.SortOrderDesc, defaultTimeField, "epoch_nanos_int")
+	b.Size(stringToIntWithDefaultValue(metric.Settings.Get("spanLimit").MustString(), 5000))
+}
+
 func processDocumentQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defaultTimeField string) {
 	metric := q.Metrics[0]
 	b.Sort(es.SortOrderDesc, defaultTimeField, "epoch_nanos_int")

pkg/quickwit/data_query_test.go

@@ -12,6 +12,18 @@ import (
 	es "github.com/quickwit-oss/quickwit-datasource/pkg/quickwit/client"
 )
 
+func TestQueryTypeHelpersDoNotPanicWithoutMetrics(t *testing.T) {
+	query := &Query{}
+
+	require.NotPanics(t, func() {
+		assert.False(t, isLogsQuery(query))
+		assert.False(t, isTracesQuery(query))
+		assert.False(t, isTraceSearchQuery(query))
+		assert.False(t, isRawDataQuery(query))
+		assert.False(t, isRawDocumentQuery(query))
+	})
+}
+
 func TestExecuteElasticsearchDataQuery(t *testing.T) {
 	from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
 	to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
@@ -1321,6 +1333,42 @@ func TestExecuteElasticsearchDataQuery(t *testing.T) {
 			require.Equal(t, sr.Size, 1000)
 		})
 
+		t.Run("With traces query should return query sorted by ascending time", func(t *testing.T) {
+			c := newFakeClient()
+			_, err := executeElasticsearchDataQuery(c, `{
+				"query": "trace_id:3c191d03fa8be0653c191d03fa8be065",
+				"metrics": [{ "type": "traces", "id": "1", "settings": { "limit": "5000" }}]
+			}`, from, to)
+			require.NoError(t, err)
+			sr := c.multisearchRequests[0][0]
+			require.Equal(t, sr.Size, 5000)
+			require.Equal(t, sr.Sort[0]["@timestamp"]["order"], "asc")
+
+			rangeFilter := sr.Query.Bool.Filters[0].(*es.DateRangeFilter)
+			require.Equal(t, rangeFilter.Lte, "2018-05-15T17:55:00Z")
+			require.Equal(t, rangeFilter.Gte, "2018-05-15T17:50:00Z")
+			queryFilter := sr.Query.Bool.Filters[1].(*es.QueryStringFilter)
+			require.Equal(t, queryFilter.Query, "trace_id:3c191d03fa8be0653c191d03fa8be065")
+		})
+
+		t.Run("With trace search query should scan spans sorted by descending time", func(t *testing.T) {
+			c := newFakeClient()
+			_, err := executeElasticsearchDataQuery(c, `{
+				"query": "service_name:quickwit",
+				"metrics": [{ "type": "trace_search", "id": "1", "settings": { "limit": "20", "spanLimit": "2500" }}]
+			}`, from, to)
+			require.NoError(t, err)
+			sr := c.multisearchRequests[0][0]
+			require.Equal(t, sr.Size, 2500)
+			require.Equal(t, sr.Sort[0]["@timestamp"]["order"], "desc")
+
+			rangeFilter := sr.Query.Bool.Filters[0].(*es.DateRangeFilter)
+			require.Equal(t, rangeFilter.Lte, "2018-05-15T17:55:00Z")
+			require.Equal(t, rangeFilter.Gte, "2018-05-15T17:50:00Z")
+			queryFilter := sr.Query.Bool.Filters[1].(*es.QueryStringFilter)
+			require.Equal(t, queryFilter.Query, "service_name:quickwit")
+		})
+
 		t.Run("With invalid query should return error", (func(t *testing.T) {
 			c := newFakeClient()
 			_, err := executeElasticsearchDataQuery(c, `{
@@ -1721,5 +1769,5 @@ func executeElasticsearchDataQuery(c es.Client, body string, from, to time.Time)
 		return &backend.QueryDataResponse{}, err
 	}
 
-	return parseResponse(res, queries, configuredFields)
+	return parseResponse(res, queries, configuredFields, nil)
 }

pkg/quickwit/elasticsearch.go

@@ -45,7 +45,7 @@ func queryData(ctx context.Context, dataQueries []backend.DataQuery, dsInfo *es.
 		return &backend.QueryDataResponse{}, err
 	}
 
-	return parseResponse(res, queries, dsInfo.ConfiguredFields)
+	return parseResponse(res, queries, dsInfo.ConfiguredFields, dsInfo)
 }
 
 func handleQuickwitErrors(err error) (*backend.QueryDataResponse, error) {

pkg/quickwit/models.go

@@ -60,6 +60,8 @@ var metricAggType = map[string]string{
 	"raw_data":       "Raw Data",
 	"rate":           "Rate",
 	"logs":           "Logs",
+	"traces":         "Traces",
+	"trace_search":   "Trace search",
 }
 
 var extendedStats = map[string]string{

pkg/quickwit/quickwit.go

@@ -67,6 +67,26 @@ func NewQuickwitDatasource(ctx context.Context, settings backend.DataSourceInsta
 		logMessageField = ""
 	}
 
+	logsDatasourceUID, ok := jsonData["logsDatasourceUid"].(string)
+	if !ok {
+		logsDatasourceUID = ""
+	}
+
+	logsDatasourceName, ok := jsonData["logsDatasourceName"].(string)
+	if !ok {
+		logsDatasourceName = ""
+	}
+
+	tracesDatasourceUID, ok := jsonData["tracesDatasourceUid"].(string)
+	if !ok {
+		tracesDatasourceUID = ""
+	}
+
+	tracesDatasourceName, ok := jsonData["tracesDatasourceName"].(string)
+	if !ok {
+		tracesDatasourceName = ""
+	}
+
 	index, ok := jsonData["index"].(string)
 	if !ok {
 		index = ""
@@ -99,6 +119,12 @@ func NewQuickwitDatasource(ctx context.Context, settings backend.DataSourceInsta
 
 	model := es.DatasourceInfo{
 		ID:                         settings.ID,
+		UID:                        settings.UID,
+		Name:                       settings.Name,
+		LogsDatasourceUID:          logsDatasourceUID,
+		LogsDatasourceName:         logsDatasourceName,
+		TracesDatasourceUID:        tracesDatasourceUID,
+		TracesDatasourceName:       tracesDatasourceName,
 		URL:                        settings.URL,
 		HTTPClient:                 httpCli,
 		Database:                   index,

pkg/quickwit/response_parser.go

@@ -41,7 +41,7 @@ const (
 
 var searchWordsRegex = regexp.MustCompile(regexp.QuoteMeta(es.HighlightPreTagsString) + `(.*?)` + regexp.QuoteMeta(es.HighlightPostTagsString))
 
-func parseResponse(rawResponses []*json.RawMessage, targets []*Query, configuredFields es.ConfiguredFields) (*backend.QueryDataResponse, error) {
+func parseResponse(rawResponses []*json.RawMessage, targets []*Query, configuredFields es.ConfiguredFields, dsInfo *es.DatasourceInfo) (*backend.QueryDataResponse, error) {
 	result := backend.QueryDataResponse{
 		Responses: backend.Responses{},
 	}
@@ -54,7 +54,7 @@ func parseResponse(rawResponses []*json.RawMessage, targets []*Query, configured
 
 		byteReader := bytes.NewReader(*rawRes)
 		dec := json.NewDecoder(byteReader)
-		if isLogsQuery(target) {
+		if isLogsQuery(target) || isTraceSearchQuery(target) || isTracesQuery(target) {
 			dec.UseNumber()
 		}
 		var res *es.SearchResponse
@@ -92,6 +92,18 @@ func parseResponse(rawResponses []*json.RawMessage, targets []*Query, configured
 				return &backend.QueryDataResponse{}, err
 			}
 			result.Responses[target.RefID] = queryRes
+		} else if isTraceSearchQuery(target) {
+			err := processTraceSearchResponse(res, target, dsInfo, &queryRes)
+			if err != nil {
+				return &backend.QueryDataResponse{}, err
+			}
+			result.Responses[target.RefID] = queryRes
+		} else if isTracesQuery(target) {
+			err := processTracesResponse(res, target, configuredFields, dsInfo, &queryRes)
+			if err != nil {
+				return &backend.QueryDataResponse{}, err
+			}
+			result.Responses[target.RefID] = queryRes
 		} else {
 			// Process as metric query result
 			props := make(map[string]string)