From 11f13f304b7606a0cf5968f04a76663e4321dd9a Mon Sep 17 00:00:00 2001 From: Gediminas Date: Mon, 4 May 2020 17:00:59 +0300 Subject: [PATCH 01/23] [m3comparator] a feature to clear data on memory storage --- src/cmd/services/m3comparator/main/series_load_handler.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cmd/services/m3comparator/main/series_load_handler.go b/src/cmd/services/m3comparator/main/series_load_handler.go index 62be51d954..f29d76ec11 100644 --- a/src/cmd/services/m3comparator/main/series_load_handler.go +++ b/src/cmd/services/m3comparator/main/series_load_handler.go @@ -188,6 +188,11 @@ func (l *seriesLoadHandler) serveHTTP(r *http.Request) error { l.Lock() defer l.Unlock() + if r.Method == http.MethodDelete { + l.nameIDSeriesMap = make(map[string]idSeriesMap) + return nil + } + logger := l.iterOpts.iOpts.Logger() body := r.Body defer body.Close() From e7090fbbcacc567c8ba920bcbb33a904be631445 Mon Sep 17 00:00:00 2001 From: Gediminas Date: Mon, 4 May 2020 17:20:54 +0300 Subject: [PATCH 02/23] test added --- .../main/series_load_handler_test.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/cmd/services/m3comparator/main/series_load_handler_test.go b/src/cmd/services/m3comparator/main/series_load_handler_test.go index 6b92e20f9d..2e41a1174e 100644 --- a/src/cmd/services/m3comparator/main/series_load_handler_test.go +++ b/src/cmd/services/m3comparator/main/series_load_handler_test.go @@ -186,6 +186,40 @@ func TestIngestSeries(t *testing.T) { assert.Equal(t, j, len(expected.Datapoints)) } +func TestClearData(t *testing.T) { + opts := iteratorOptions{ + encoderPool: encoderPool, + iteratorPools: iterPools, + tagOptions: tagOptions, + iOpts: iOpts, + } + + req, err := http.NewRequest(http.MethodPost, "", strings.NewReader(seriesStr)) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + + handler := newSeriesLoadHandler(opts) + handler.ServeHTTP(recorder, req) + + assert.Equal(t, http.StatusOK, recorder.Code) + + iters, err := handler.getSeriesIterators("series_name") + require.NoError(t, err) + require.Equal(t, 1, len(iters.Iters())) + + // Call clear data + req, err = http.NewRequest(http.MethodDelete, "", nil) + require.NoError(t, err) + + handler.ServeHTTP(recorder, req) + assert.Equal(t, http.StatusOK, recorder.Code) + + iters, err = handler.getSeriesIterators("series_name") + require.NoError(t, err) + require.Nil(t, iters) +} + func readTags(it encoding.SeriesIterator) parser.Tags { tagIter := it.Tags() tags := make(parser.Tags, tagIter.Len()) From 6aadbf1bfd2b953b1587104e9f1b008836b9c49d Mon Sep 17 00:00:00 2001 From: linasm Date: Wed, 6 May 2020 11:48:23 +0300 Subject: [PATCH 03/23] [comparator] Generate random multi series metrics --- src/cmd/services/m3comparator/main/main.go | 2 +- .../m3comparator/main/parser/parser.go | 6 +- src/cmd/services/m3comparator/main/querier.go | 100 ++++++-- .../m3comparator/main/querier_test.go | 218 ++++++++++++++++++ .../m3comparator/main/series_load_handler.go | 23 +- .../main/series_load_handler_test.go | 2 +- 6 files changed, 323 insertions(+), 28 deletions(-) create mode 100644 src/cmd/services/m3comparator/main/querier_test.go diff --git a/src/cmd/services/m3comparator/main/main.go b/src/cmd/services/m3comparator/main/main.go index 5d190114d3..89e771e8a0 100644 --- a/src/cmd/services/m3comparator/main/main.go +++ b/src/cmd/services/m3comparator/main/main.go @@ -76,7 +76,7 @@ func main() { iOpts: iOpts, } - seriesLoader := newSeriesLoadHandler(opts) + seriesLoader := newHTTPSeriesLoadHandler(opts) querier := &querier{opts: opts, handler: seriesLoader} server := remote.NewGRPCServer( querier, diff --git a/src/cmd/services/m3comparator/main/parser/parser.go b/src/cmd/services/m3comparator/main/parser/parser.go index f2119aeba7..1c5902b30d 100644 --- a/src/cmd/services/m3comparator/main/parser/parser.go +++ b/src/cmd/services/m3comparator/main/parser/parser.go @@ -29,7 +29,7 @@ import ( "time" ) -// Series is a flat JSON serieazeable representation of the series. +// Series is a flat JSON serializable representation of the series. type Series struct { id string @@ -39,7 +39,7 @@ type Series struct { Datapoints Datapoints `json:"datapoints"` } -// Tag is a simple JSON serieazeable representation of a tag. +// Tag is a simple JSON serializable representation of a tag. type Tag [2]string // NewTag creates a new tag with a given name and value. @@ -57,7 +57,7 @@ func (t Tag) Value() string { return t[1] } -// Tags is a simple JSON serieazeable representation of tags. +// Tags is a simple JSON serializable representation of tags. type Tags []Tag // Get returns a list of tag values with the given name. diff --git a/src/cmd/services/m3comparator/main/querier.go b/src/cmd/services/m3comparator/main/querier.go index a89b246bf1..accc81883e 100644 --- a/src/cmd/services/m3comparator/main/querier.go +++ b/src/cmd/services/m3comparator/main/querier.go @@ -26,7 +26,9 @@ import ( "fmt" "math" "math/rand" + "regexp" "strconv" + "strings" "sync" "time" @@ -43,14 +45,14 @@ var _ m3.Querier = (*querier)(nil) type querier struct { opts iteratorOptions - handler *seriesLoadHandler + handler seriesLoadHandler sync.Mutex } func noop() error { return nil } type seriesBlock []ts.Datapoint -type tagMap map[string]string + type series struct { blocks []seriesBlock tags parser.Tags @@ -129,8 +131,14 @@ func (q *querier) FetchCompressed( } } + const blockSize = time.Hour * 12 + if iters == nil || iters.Len() == 0 { - iters, err = q.generateRandomIters(query) + series, err := q.generateRandomSeries(query, blockSize) + if err != nil { + return m3.SeriesFetchResult{}, noop, err + } + iters, err = buildSeriesIterators(series, query.Start, blockSize, q.opts) if err != nil { return m3.SeriesFetchResult{}, noop, err } @@ -147,15 +155,40 @@ func (q *querier) FetchCompressed( }, cleanup, nil } -func (q *querier) generateRandomIters( +func (q *querier) generateRandomSeries( query *storage.FetchQuery, -) (encoding.SeriesIterators, error) { + blockSize time.Duration, +) ([]series, error) { var ( - blockSize = time.Hour * 12 - start = query.Start.Truncate(blockSize) - end = query.End.Truncate(blockSize).Add(blockSize) - opts = q.opts - gens = []seriesGen{ + start = query.Start.Truncate(blockSize) + end = query.End.Truncate(blockSize).Add(blockSize) + ) + + multiSeriesMetrics := "" + for _, matcher := range query.TagMatchers { + // filter if name, otherwise return all. + if bytes.Equal(q.opts.tagOptions.MetricName(), matcher.Name) { + if matched, _ := regexp.Match(`^multi_\d+$`, matcher.Value); matched { + multiSeriesMetrics = string(matcher.Value) + } + } + } + + if multiSeriesMetrics != "" { + return q.generateMultiSeriesMetrics(multiSeriesMetrics, start, end, time.Second*30, blockSize) + } + + return q.generateSingleSeriesMetrics(query, start, end, blockSize) +} + +func (q *querier) generateSingleSeriesMetrics( + query *storage.FetchQuery, + start time.Time, + end time.Time, + blockSize time.Duration, +) ([]series, error) { + var ( + gens = []seriesGen{ {"foo", time.Second}, {"bar", time.Second * 15}, {"quail", time.Minute}, @@ -169,7 +202,7 @@ func (q *querier) generateRandomIters( rand.Seed(start.Unix()) for _, matcher := range query.TagMatchers { // filter if name, otherwise return all. - if bytes.Equal(opts.tagOptions.MetricName(), matcher.Name) { + if bytes.Equal(q.opts.tagOptions.MetricName(), matcher.Name) { value := string(matcher.Value) for _, gen := range gens { if value == gen.name { @@ -204,10 +237,10 @@ func (q *querier) generateRandomIters( seriesList := make([]series, 0, len(actualGens)) for _, gen := range actualGens { - tags := parser.Tags{ + tags := parser.Tags{ parser.NewTag("__name__", gen.name), - parser.NewTag("foobar", "qux"), - parser.NewTag("name", gen.name), + parser.NewTag("foobar", "qux"), + parser.NewTag("name", gen.name), } series, err := generateSeries(start, end, blockSize, gen.res, tags) @@ -218,7 +251,44 @@ func (q *querier) generateRandomIters( seriesList = append(seriesList, series) } - return buildSeriesIterators(seriesList, query.Start, blockSize, opts) + return seriesList, nil +} + +func (q *querier) generateMultiSeriesMetrics( + metricsName string, + start time.Time, + end time.Time, + resolution time.Duration, + blockSize time.Duration, +) ([]series, error) { + suffix := strings.TrimPrefix(metricsName, "multi_") + seriesCount, err := strconv.Atoi(suffix) + if err != nil { + return nil, err + } + + q.Lock() + defer q.Unlock() + rand.Seed(start.Unix()) + + seriesList := make([]series, seriesCount) + for i := 0; i < seriesCount; i++ { + tags := parser.Tags{ + parser.NewTag("__name__", metricsName), + parser.NewTag("id", strconv.Itoa(i)), + parser.NewTag("parity", strconv.Itoa(i%2)), + parser.NewTag("const", "x"), + } + + series, err := generateSeries(start, end, blockSize, resolution, tags) + if err != nil { + return nil, err + } + + seriesList[i] = series + } + + return seriesList, nil } // SearchCompressed fetches matching tags based on a query. diff --git a/src/cmd/services/m3comparator/main/querier_test.go b/src/cmd/services/m3comparator/main/querier_test.go new file mode 100644 index 0000000000..a449201def --- /dev/null +++ b/src/cmd/services/m3comparator/main/querier_test.go @@ -0,0 +1,218 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +import ( + "testing" + "time" + + "github.com/m3db/m3/src/dbnode/encoding" + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" + xtest "github.com/m3db/m3/src/x/test" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" +) + +type testSeriesLoadHandler struct { + iters encoding.SeriesIterators +} + +func (h *testSeriesLoadHandler) getSeriesIterators(name string) (encoding.SeriesIterators, error) { + return h.iters, nil +} + +var _ seriesLoadHandler = (*testSeriesLoadHandler)(nil) + +type tagMap map[string]string + +var ( + iteratorOpts = iteratorOptions{ + encoderPool: encoderPool, + iteratorPools: iterPools, + tagOptions: tagOptions, + iOpts: iOpts, + } + metricNameTag = string(iteratorOpts.tagOptions.MetricName()) +) + +func TestFetchCompressedReturnsPreloadedData(t *testing.T) { + ctrl := xtest.NewController(t) + defer ctrl.Finish() + + predefinedSeriesCount := 100 + iters := encoding.NewMockSeriesIterators(ctrl) + iters.EXPECT().Len().Return(predefinedSeriesCount).MinTimes(1) + iters.EXPECT().Close() + + seriesLoader := &testSeriesLoadHandler{iters} + + querier := &querier{opts: iteratorOpts, handler: seriesLoader} + + query := matcherQuery(t, metricNameTag, "preloaded") + + result, cleanup, err := querier.FetchCompressed(nil, query, nil) + assert.NoError(t, err) + defer cleanup() + + assert.Equal(t, predefinedSeriesCount, result.SeriesIterators.Len()) +} + +func TestFetchCompressedGeneratesRandomData(t *testing.T) { + tests := []struct { + name string + givenQuery *storage.FetchQuery + wantSeries []tagMap + }{ + { + name: "random data for known metrics", + givenQuery: matcherQuery(t, metricNameTag, "quail"), + wantSeries: []tagMap{ + { + metricNameTag: "quail", + "foobar": "qux", + "name": "quail", + }, + }, + }, + { + name: "a hardcoded list of metrics", + givenQuery: matcherQuery(t, metricNameTag, "unknown"), + wantSeries: []tagMap{ + { + metricNameTag: "foo", + "foobar": "qux", + "name": "foo", + }, + { + metricNameTag: "bar", + "foobar": "qux", + "name": "bar", + }, + { + metricNameTag: "quail", + "foobar": "qux", + "name": "quail", + }, + }, + }, + { + name: "a given number of single series metrics", + givenQuery: matcherQuery(t, "gen", "2"), + wantSeries: []tagMap{ + { + metricNameTag: "foo_0", + "foobar": "qux", + "name": "foo_0", + }, + { + metricNameTag: "foo_1", + "foobar": "qux", + "name": "foo_1", + }, + }, + }, + { + name: "single metrics with a given number of series", + givenQuery: matcherQuery(t, metricNameTag, "multi_4"), + wantSeries: []tagMap{ + { + metricNameTag: "multi_4", + "const": "x", + "id": "0", + "parity": "0", + }, + { + metricNameTag: "multi_4", + "const": "x", + "id": "1", + "parity": "1", + }, + { + metricNameTag: "multi_4", + "const": "x", + "id": "2", + "parity": "0", + }, + { + metricNameTag: "multi_4", + "const": "x", + "id": "3", + "parity": "1", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + querier := &querier{opts: iteratorOpts, handler: emptySeriesLoader(ctrl)} + + result, cleanup, err := querier.FetchCompressed(nil, tt.givenQuery, nil) + assert.NoError(t, err) + defer cleanup() + + assert.Equal(t, len(tt.wantSeries), result.SeriesIterators.Len()) + for i, expectedTags := range tt.wantSeries { + iter := result.SeriesIterators.Iters()[i] + assert.Equal(t, expectedTags, extractTags(iter)) + assert.True(t, iter.Next(), "Must have some datapoints generated.") + } + }) + } +} + +func matcherQuery(t *testing.T, matcherName, matcherValue string) *storage.FetchQuery { + matcher, err := models.NewMatcher(models.MatchEqual, []byte(matcherName), []byte(matcherValue)) + assert.NoError(t, err) + + now := time.Now() + + return &storage.FetchQuery{ + TagMatchers: []models.Matcher{matcher}, + Start: now.Add(-time.Hour), + End: now, + } +} + +func emptySeriesLoader(ctrl *gomock.Controller) seriesLoadHandler { + iters := encoding.NewMockSeriesIterators(ctrl) + iters.EXPECT().Len().Return(0).AnyTimes() + + return &testSeriesLoadHandler{iters} +} + +func extractTags(seriesIter encoding.SeriesIterator) tagMap { + tagsIter := seriesIter.Tags().Duplicate() + defer tagsIter.Close() + + tags := make(tagMap) + for tagsIter.Next() { + tag := tagsIter.Current() + tags[tag.Name.String()] = tag.Value.String() + } + + return tags +} diff --git a/src/cmd/services/m3comparator/main/series_load_handler.go b/src/cmd/services/m3comparator/main/series_load_handler.go index 560321cc05..bc05b813bf 100644 --- a/src/cmd/services/m3comparator/main/series_load_handler.go +++ b/src/cmd/services/m3comparator/main/series_load_handler.go @@ -47,24 +47,31 @@ type idSeriesMap struct { type nameIDSeriesMap map[string]idSeriesMap -type seriesLoadHandler struct { +type seriesLoadHandler interface { + getSeriesIterators(string) (encoding.SeriesIterators, error) +} + +type httpSeriesLoadHandler struct { sync.RWMutex nameIDSeriesMap nameIDSeriesMap iterOpts iteratorOptions } -var _ http.Handler = (*seriesLoadHandler)(nil) +var ( + _ http.Handler = (*httpSeriesLoadHandler)(nil) + _ seriesLoadHandler = (*httpSeriesLoadHandler)(nil) +) -// newSeriesLoadHandler builds a handler that can load series +// newHTTPSeriesLoadHandler builds a handler that can load series // to the comparator via an http endpoint. -func newSeriesLoadHandler(iterOpts iteratorOptions) *seriesLoadHandler { - return &seriesLoadHandler{ +func newHTTPSeriesLoadHandler(iterOpts iteratorOptions) *httpSeriesLoadHandler { + return &httpSeriesLoadHandler{ iterOpts: iterOpts, nameIDSeriesMap: make(nameIDSeriesMap), } } -func (l *seriesLoadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (l *httpSeriesLoadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { logger := l.iterOpts.iOpts.Logger() err := l.serveHTTP(r) if err != nil { @@ -76,7 +83,7 @@ func (l *seriesLoadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (l *seriesLoadHandler) getSeriesIterators( +func (l *httpSeriesLoadHandler) getSeriesIterators( name string) (encoding.SeriesIterators, error) { l.RLock() defer l.RUnlock() @@ -185,7 +192,7 @@ func calculateSeriesRange(seriesList []parser.Series) (time.Time, time.Time) { return start, end } -func (l *seriesLoadHandler) serveHTTP(r *http.Request) error { +func (l *httpSeriesLoadHandler) serveHTTP(r *http.Request) error { l.Lock() defer l.Unlock() diff --git a/src/cmd/services/m3comparator/main/series_load_handler_test.go b/src/cmd/services/m3comparator/main/series_load_handler_test.go index d007232756..46b21a77c8 100644 --- a/src/cmd/services/m3comparator/main/series_load_handler_test.go +++ b/src/cmd/services/m3comparator/main/series_load_handler_test.go @@ -159,7 +159,7 @@ func TestIngestSeries(t *testing.T) { recorder := httptest.NewRecorder() - handler := newSeriesLoadHandler(opts) + handler := newHTTPSeriesLoadHandler(opts) handler.ServeHTTP(recorder, req) assert.Equal(t, http.StatusOK, recorder.Code) From efb837d6e193e01e7cd2ef428d1300fa6d033e22 Mon Sep 17 00:00:00 2001 From: linasm Date: Wed, 6 May 2020 12:16:39 +0300 Subject: [PATCH 04/23] Remove unnecessary comment --- src/cmd/services/m3comparator/main/querier.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cmd/services/m3comparator/main/querier.go b/src/cmd/services/m3comparator/main/querier.go index accc81883e..11ef0fdea6 100644 --- a/src/cmd/services/m3comparator/main/querier.go +++ b/src/cmd/services/m3comparator/main/querier.go @@ -166,7 +166,6 @@ func (q *querier) generateRandomSeries( multiSeriesMetrics := "" for _, matcher := range query.TagMatchers { - // filter if name, otherwise return all. if bytes.Equal(q.opts.tagOptions.MetricName(), matcher.Name) { if matched, _ := regexp.Match(`^multi_\d+$`, matcher.Value); matched { multiSeriesMetrics = string(matcher.Value) From 2a261c9a5e4f0035d9c81d8ab8b352bd79b3e602 Mon Sep 17 00:00:00 2001 From: linasm Date: Wed, 6 May 2020 16:17:09 +0300 Subject: [PATCH 05/23] Address review feedback --- src/cmd/services/m3comparator/main/querier.go | 24 +++++++++---------- .../m3comparator/main/querier_test.go | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/cmd/services/m3comparator/main/querier.go b/src/cmd/services/m3comparator/main/querier.go index 11ef0fdea6..9577297d04 100644 --- a/src/cmd/services/m3comparator/main/querier.go +++ b/src/cmd/services/m3comparator/main/querier.go @@ -164,19 +164,15 @@ func (q *querier) generateRandomSeries( end = query.End.Truncate(blockSize).Add(blockSize) ) - multiSeriesMetrics := "" + metricNameTag := q.opts.tagOptions.MetricName() for _, matcher := range query.TagMatchers { - if bytes.Equal(q.opts.tagOptions.MetricName(), matcher.Name) { + if bytes.Equal(metricNameTag, matcher.Name) { if matched, _ := regexp.Match(`^multi_\d+$`, matcher.Value); matched { - multiSeriesMetrics = string(matcher.Value) + return q.generateMultiSeriesMetrics(string(matcher.Value), start, end, time.Second*30, blockSize) } } } - if multiSeriesMetrics != "" { - return q.generateMultiSeriesMetrics(multiSeriesMetrics, start, end, time.Second*30, blockSize) - } - return q.generateSingleSeriesMetrics(query, start, end, blockSize) } @@ -199,9 +195,11 @@ func (q *querier) generateSingleSeriesMetrics( q.Lock() defer q.Unlock() rand.Seed(start.Unix()) + + metricNameTag := q.opts.tagOptions.MetricName() for _, matcher := range query.TagMatchers { // filter if name, otherwise return all. - if bytes.Equal(q.opts.tagOptions.MetricName(), matcher.Name) { + if bytes.Equal(metricNameTag, matcher.Name) { value := string(matcher.Value) for _, gen := range gens { if value == gen.name { @@ -218,12 +216,12 @@ func (q *querier) generateSingleSeriesMetrics( return nil, err } - actualGens = make([]seriesGen, count) + actualGens = make([]seriesGen, 0, count) for i := 0; i < count; i++ { - actualGens[i] = seriesGen{ + actualGens = append(actualGens, seriesGen{ res: time.Second * 15, name: fmt.Sprintf("foo_%d", i), - } + }) } break @@ -270,7 +268,7 @@ func (q *querier) generateMultiSeriesMetrics( defer q.Unlock() rand.Seed(start.Unix()) - seriesList := make([]series, seriesCount) + seriesList := make([]series, 0, seriesCount) for i := 0; i < seriesCount; i++ { tags := parser.Tags{ parser.NewTag("__name__", metricsName), @@ -284,7 +282,7 @@ func (q *querier) generateMultiSeriesMetrics( return nil, err } - seriesList[i] = series + seriesList = append(seriesList, series) } return seriesList, nil diff --git a/src/cmd/services/m3comparator/main/querier_test.go b/src/cmd/services/m3comparator/main/querier_test.go index a449201def..b62ce60cb2 100644 --- a/src/cmd/services/m3comparator/main/querier_test.go +++ b/src/cmd/services/m3comparator/main/querier_test.go @@ -165,7 +165,7 @@ func TestFetchCompressedGeneratesRandomData(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) + ctrl := xtest.NewController(t) defer ctrl.Finish() querier := &querier{opts: iteratorOpts, handler: emptySeriesLoader(ctrl)} From 5ec9d4098a18cbebc1ca9bdf74372a250f1f1ccf Mon Sep 17 00:00:00 2001 From: linasm Date: Wed, 6 May 2020 17:05:31 +0300 Subject: [PATCH 06/23] [comparator] Filter query results by tag matchers --- .../services/m3comparator/main/filterer.go | 95 +++++++ .../m3comparator/main/filterer_test.go | 266 ++++++++++++++++++ src/cmd/services/m3comparator/main/querier.go | 34 ++- .../m3comparator/main/querier_test.go | 68 ++++- 4 files changed, 448 insertions(+), 15 deletions(-) create mode 100644 src/cmd/services/m3comparator/main/filterer.go create mode 100644 src/cmd/services/m3comparator/main/filterer_test.go diff --git a/src/cmd/services/m3comparator/main/filterer.go b/src/cmd/services/m3comparator/main/filterer.go new file mode 100644 index 0000000000..ecda2bc85e --- /dev/null +++ b/src/cmd/services/m3comparator/main/filterer.go @@ -0,0 +1,95 @@ +package main + +import ( + "bytes" + "fmt" + "regexp" + + "github.com/m3db/m3/src/dbnode/encoding" + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/x/ident" +) + +func filter(series encoding.SeriesIterators, tagMatchers models.Matchers) encoding.SeriesIterators { + var filtered []encoding.SeriesIterator + + for _, iter := range series.Iters() { + if matchesAll(tagMatchers, iter.Tags()) { + filtered = append(filtered, iter) + } + } + + return encoding.NewSeriesIterators(filtered, nil) +} + +func matchesAll(tagMatchers models.Matchers, tagsIter ident.TagIterator) bool { + for _, tagMatcher := range tagMatchers { + if !isIgnored(tagMatcher) && !matchesOne(tagMatcher, tagsIter) { + return false + } + } + + return true +} + +func matchesOne(tagMatcher models.Matcher, tagsIter ident.TagIterator) bool { + tag := findTag(tagMatcher.Name, tagsIter) + + return matches(tagMatcher, tag) +} + +func isIgnored(tagMatcher models.Matcher) bool { + return tagMatcher.String() == `role="remote"` // this matcher gets injected by Prometheus +} + +func findTag(name []byte, tagsIter ident.TagIterator) ident.Tag { + tagsIter = tagsIter.Duplicate() + defer tagsIter.Close() + + for tagsIter.Next() { + tag := tagsIter.Current() + if bytes.Equal(tag.Name.Bytes(), name) { + return tag + } + } + + return ident.StringTag("", "") +} + +func matches(tagMatcher models.Matcher, tag ident.Tag) bool { + var ( + name = tag.Name.Bytes() + value = tag.Value.Bytes() + invert = false + ) + + switch tagMatcher.Type { + + case models.MatchNotEqual: + invert = true + fallthrough + + case models.MatchEqual: + return bytes.Equal(tagMatcher.Value, value) != invert + + case models.MatchNotRegexp: + invert = true + fallthrough + + case models.MatchRegexp: + m, _ := regexp.Match(fmt.Sprintf("^%s$", tagMatcher.Value), value) + return m != invert + + case models.MatchNotField: + invert = true + fallthrough + + case models.MatchField: + return bytes.Equal(tagMatcher.Name, name) != invert + + case models.MatchAll: + return true + } + + return false +} diff --git a/src/cmd/services/m3comparator/main/filterer_test.go b/src/cmd/services/m3comparator/main/filterer_test.go new file mode 100644 index 0000000000..e2b4076c1e --- /dev/null +++ b/src/cmd/services/m3comparator/main/filterer_test.go @@ -0,0 +1,266 @@ +// Copyright (c) 2020 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +import ( + "github.com/m3db/m3/src/cmd/services/m3comparator/main/parser" + "testing" + "time" + + "github.com/m3db/m3/src/dbnode/encoding" + "github.com/m3db/m3/src/query/models" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + series1 = parser.Tags{ + parser.NewTag("foo", "bar"), + parser.NewTag("baz", "quix"), + } + + series2 = parser.Tags{ + parser.NewTag("alpha", "a"), + parser.NewTag("beta", "b"), + } + + allSeries = list(series1, series2) +) + +func TestFilter(t *testing.T) { + testCases := []struct { + name string + givenMatchers models.Matchers + wantedSeries []series + }{ + { + name: "No matchers", + givenMatchers: models.Matchers{}, + wantedSeries: list(series1, series2), + }, + + { + name: "MatchEqual on one tag", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchEqual, "bar"), + }, + wantedSeries: list(series1), + }, + { + name: "MatchEqual on two tags", + givenMatchers: models.Matchers{ + tagMatcher("alpha", models.MatchEqual, "a"), + tagMatcher("beta", models.MatchEqual, "b"), + }, + wantedSeries: list(series2), + }, + { + name: "Two MatchEqual excluding every series each", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchEqual, "bar"), + tagMatcher("beta", models.MatchEqual, "b"), + }, + wantedSeries: list(), + }, + { + name: "MatchEqual excluding all series", + givenMatchers: models.Matchers{ + tagMatcher("unknown", models.MatchEqual, "whatever"), + }, + wantedSeries: list(), + }, + { + name: "MatchEqual on empty value", + givenMatchers: models.Matchers{ + tagMatcher("unknown", models.MatchEqual, ""), + }, + wantedSeries: list(series1, series2), + }, + + { + name: "MatchNotEqual on one tag", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotEqual, "bar"), + }, + wantedSeries: list(series2), + }, + { + name: "MatchNotEqual on two tags", + givenMatchers: models.Matchers{ + tagMatcher("alpha", models.MatchNotEqual, "a"), + tagMatcher("beta", models.MatchNotEqual, "b"), + }, + wantedSeries: list(series1), + }, + { + name: "Two MatchNotEqual excluding every series each", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotEqual, "bar"), + tagMatcher("beta", models.MatchNotEqual, "b"), + }, + wantedSeries: list(), + }, + { + name: "MatchNotEqual accepting all series", + givenMatchers: models.Matchers{ + tagMatcher("unknown", models.MatchNotEqual, "whatever"), + }, + wantedSeries: list(series1, series2), + }, + { + name: "MatchNotEqual on empty value", + givenMatchers: models.Matchers{ + tagMatcher("unknown", models.MatchNotEqual, ""), + }, + wantedSeries: list(), + }, + + { + name: "MatchRegexp on full value", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchRegexp, "bar"), + }, + wantedSeries: list(series1), + }, + { + name: "MatchRegexp with wildcard", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchRegexp, "b.+"), + }, + wantedSeries: list(series1), + }, + { + name: "MatchRegexp with alternatives", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchRegexp, "bax|bar|baz"), + }, + wantedSeries: list(series1), + }, + { + name: "MatchRegexp unmatched", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchRegexp, "bax|baz"), + }, + wantedSeries: list(), + }, + + { + name: "MatchNotRegexp on full value", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotRegexp, "bar"), + }, + wantedSeries: list(series2), + }, + { + name: "MatchNotRegexp with wildcard", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotRegexp, "b.+"), + }, + wantedSeries: list(series2), + }, + { + name: "MatchNotRegexp with alternatives", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotRegexp, "bax|bar|baz"), + }, + wantedSeries: list(series2), + }, + { + name: "MatchNotRegexp matching all", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotRegexp, "bax|baz"), + }, + wantedSeries: list(series1, series2), + }, + + { + name: "MatchField", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchField, ""), + }, + wantedSeries: list(series1), + }, + + { + name: "MatchNotField", + givenMatchers: models.Matchers{ + tagMatcher("foo", models.MatchNotField, ""), + }, + wantedSeries: list(series2), + }, + + { + name: `Ignore 'role="remote"' matcher added by Prometheus`, + givenMatchers: models.Matchers{ + tagMatcher("role", models.MatchEqual, "remote"), + }, + wantedSeries: list(series1, series2), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + unfilteredIters, err := toSeriesIterators(allSeries) + require.NoError(t, err) + filteredIters := filter(unfilteredIters, tc.givenMatchers) + filteredSeries := fromSeriesIterators(filteredIters) + assert.Equal(t, tc.wantedSeries, filteredSeries) + }) + } +} + +func tagMatcher(tag string, matchType models.MatchType, value string) models.Matcher { + return models.Matcher{ + Type: matchType, + Name: []byte(tag), + Value: []byte(value), + } +} + +func list(tagsList ...parser.Tags) []series { + list := make([]series, 0, len(tagsList)) + + for _, tags := range tagsList { + list = append(list, series{tags: tags}) + } + + return list +} + +func toSeriesIterators(series []series) (encoding.SeriesIterators, error) { + return buildSeriesIterators(series, time.Now(), time.Hour, iteratorOpts) +} + +func fromSeriesIterators(seriesIters encoding.SeriesIterators) []series { + result := make([]series, 0, seriesIters.Len()) + for _, iter := range seriesIters.Iters() { + tagsIter := iter.Tags() + tags := make(parser.Tags, 0, tagsIter.Len()) + for tagsIter.Next() { + tag := tagsIter.Current() + tags = append(tags, parser.NewTag(tag.Name.String(), tag.Value.String())) + } + result = append(result, series{tags: tags}) + } + + return result +} diff --git a/src/cmd/services/m3comparator/main/querier.go b/src/cmd/services/m3comparator/main/querier.go index 9577297d04..9f7570dcc7 100644 --- a/src/cmd/services/m3comparator/main/querier.go +++ b/src/cmd/services/m3comparator/main/querier.go @@ -115,8 +115,10 @@ func (q *querier) FetchCompressed( options *storage.FetchOptions, ) (m3.SeriesFetchResult, m3.Cleanup, error) { var ( - iters encoding.SeriesIterators - err error + iters encoding.SeriesIterators + randomSeries []series + ignoreFilter bool + err error ) name := q.opts.tagOptions.MetricName() @@ -134,16 +136,31 @@ func (q *querier) FetchCompressed( const blockSize = time.Hour * 12 if iters == nil || iters.Len() == 0 { - series, err := q.generateRandomSeries(query, blockSize) + randomSeries, ignoreFilter, err = q.generateRandomSeries(query, blockSize) if err != nil { return m3.SeriesFetchResult{}, noop, err } - iters, err = buildSeriesIterators(series, query.Start, blockSize, q.opts) + iters, err = buildSeriesIterators(randomSeries, query.Start, blockSize, q.opts) if err != nil { return m3.SeriesFetchResult{}, noop, err } } + if !ignoreFilter { + filteredIters := filter(iters, query.TagMatchers) + + cleanup := func() error { + iters.Close() + filteredIters.Close() + return nil + } + + return m3.SeriesFetchResult{ + SeriesIterators: filteredIters, + Metadata: block.NewResultMetadata(), + }, cleanup, nil + } + cleanup := func() error { iters.Close() return nil @@ -158,7 +175,7 @@ func (q *querier) FetchCompressed( func (q *querier) generateRandomSeries( query *storage.FetchQuery, blockSize time.Duration, -) ([]series, error) { +) (series []series, ignoreFilter bool, err error) { var ( start = query.Start.Truncate(blockSize) end = query.End.Truncate(blockSize).Add(blockSize) @@ -168,12 +185,15 @@ func (q *querier) generateRandomSeries( for _, matcher := range query.TagMatchers { if bytes.Equal(metricNameTag, matcher.Name) { if matched, _ := regexp.Match(`^multi_\d+$`, matcher.Value); matched { - return q.generateMultiSeriesMetrics(string(matcher.Value), start, end, time.Second*30, blockSize) + series, err = q.generateMultiSeriesMetrics(string(matcher.Value), start, end, time.Second*30, blockSize) + return } } } - return q.generateSingleSeriesMetrics(query, start, end, blockSize) + ignoreFilter = true + series, err = q.generateSingleSeriesMetrics(query, start, end, blockSize) + return } func (q *querier) generateSingleSeriesMetrics( diff --git a/src/cmd/services/m3comparator/main/querier_test.go b/src/cmd/services/m3comparator/main/querier_test.go index b62ce60cb2..def60719cd 100644 --- a/src/cmd/services/m3comparator/main/querier_test.go +++ b/src/cmd/services/m3comparator/main/querier_test.go @@ -27,10 +27,13 @@ import ( "github.com/m3db/m3/src/dbnode/encoding" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/x/ident" xtest "github.com/m3db/m3/src/x/test" + xtime "github.com/m3db/m3/src/x/time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type testSeriesLoadHandler struct { @@ -59,16 +62,37 @@ func TestFetchCompressedReturnsPreloadedData(t *testing.T) { ctrl := xtest.NewController(t) defer ctrl.Finish() - predefinedSeriesCount := 100 - iters := encoding.NewMockSeriesIterators(ctrl) - iters.EXPECT().Len().Return(predefinedSeriesCount).MinTimes(1) - iters.EXPECT().Close() + const ( + metricsName = "preloaded" + predefinedSeriesCount = 10 + ) - seriesLoader := &testSeriesLoadHandler{iters} + query := matcherQuery(t, metricNameTag, metricsName) - querier := &querier{opts: iteratorOpts, handler: seriesLoader} + metricsTag := ident.NewTags(ident.Tag{ + Name: ident.BytesID(tagOptions.MetricName()), + Value: ident.BytesID(metricsName), + }) + + iters := make([]encoding.SeriesIterator, 0, predefinedSeriesCount) + for i := 0; i < predefinedSeriesCount; i++ { + iters = append(iters, encoding.NewSeriesIterator( + encoding.SeriesIteratorOptions{ + Namespace: ident.StringID("ns"), + Tags: ident.NewTagsIterator(metricsTag), + StartInclusive: xtime.ToUnixNano(query.Start), + EndExclusive: xtime.ToUnixNano(query.End), + }, nil)) + } - query := matcherQuery(t, metricNameTag, "preloaded") + seriesIterators := encoding.NewMockSeriesIterators(ctrl) + seriesIterators.EXPECT().Len().Return(predefinedSeriesCount).MinTimes(1) + seriesIterators.EXPECT().Iters().Return(iters).Times(1) + seriesIterators.EXPECT().Close() + + seriesLoader := &testSeriesLoadHandler{seriesIterators} + + querier := &querier{opts: iteratorOpts, handler: seriesLoader} result, cleanup, err := querier.FetchCompressed(nil, query, nil) assert.NoError(t, err) @@ -161,6 +185,26 @@ func TestFetchCompressedGeneratesRandomData(t *testing.T) { }, }, }, + { + name: "apply tag filter", + givenQuery: and( + matcherQuery(t, metricNameTag, "multi_5"), + matcherQuery(t, "parity", "1")), + wantSeries: []tagMap{ + { + metricNameTag: "multi_5", + "const": "x", + "id": "1", + "parity": "1", + }, + { + metricNameTag: "multi_5", + "const": "x", + "id": "3", + "parity": "1", + }, + }, + }, } for _, tt := range tests { @@ -174,7 +218,7 @@ func TestFetchCompressedGeneratesRandomData(t *testing.T) { assert.NoError(t, err) defer cleanup() - assert.Equal(t, len(tt.wantSeries), result.SeriesIterators.Len()) + require.Equal(t, len(tt.wantSeries), result.SeriesIterators.Len()) for i, expectedTags := range tt.wantSeries { iter := result.SeriesIterators.Iters()[i] assert.Equal(t, expectedTags, extractTags(iter)) @@ -197,6 +241,14 @@ func matcherQuery(t *testing.T, matcherName, matcherValue string) *storage.Fetch } } +func and(query1, query2 *storage.FetchQuery) *storage.FetchQuery { + return &storage.FetchQuery{ + TagMatchers: append(query1.TagMatchers, query2.TagMatchers...), + Start: query1.Start, + End: query1.End, + } +} + func emptySeriesLoader(ctrl *gomock.Controller) seriesLoadHandler { iters := encoding.NewMockSeriesIterators(ctrl) iters.EXPECT().Len().Return(0).AnyTimes() From 0e58657193765346cc678bb49b78b04b39f06ef6 Mon Sep 17 00:00:00 2001 From: Gediminas Date: Tue, 12 May 2020 15:09:02 +0300 Subject: [PATCH 07/23] [m3comparator] Querying without metric name Allows to run queries like {tag="test"}. Previously only querying with metric name like random{tag="test"} was only supported. --- src/cmd/services/m3comparator/main/querier.go | 9 ++ .../m3comparator/main/querier_test.go | 102 ++++++++++++------ .../m3comparator/main/series_load_handler.go | 26 ++++- .../main/series_load_handler_test.go | 72 +++++++++++-- 4 files changed, 171 insertions(+), 38 deletions(-) diff --git a/src/cmd/services/m3comparator/main/querier.go b/src/cmd/services/m3comparator/main/querier.go index 9f7570dcc7..aab21e26c2 100644 --- a/src/cmd/services/m3comparator/main/querier.go +++ b/src/cmd/services/m3comparator/main/querier.go @@ -119,11 +119,13 @@ func (q *querier) FetchCompressed( randomSeries []series ignoreFilter bool err error + nameTagFound bool ) name := q.opts.tagOptions.MetricName() for _, matcher := range query.TagMatchers { if bytes.Equal(name, matcher.Name) { + nameTagFound = true iters, err = q.handler.getSeriesIterators(string(matcher.Value)) if err != nil { return m3.SeriesFetchResult{}, noop, err @@ -135,6 +137,13 @@ func (q *querier) FetchCompressed( const blockSize = time.Hour * 12 + if iters == nil && !nameTagFound && len(query.TagMatchers) > 0 { + iters, err = q.handler.getSeriesIterators("") + if err != nil { + return m3.SeriesFetchResult{}, noop, err + } + } + if iters == nil || iters.Len() == 0 { randomSeries, ignoreFilter, err = q.generateRandomSeries(query, blockSize) if err != nil { diff --git a/src/cmd/services/m3comparator/main/querier_test.go b/src/cmd/services/m3comparator/main/querier_test.go index def60719cd..1242f10411 100644 --- a/src/cmd/services/m3comparator/main/querier_test.go +++ b/src/cmd/services/m3comparator/main/querier_test.go @@ -58,47 +58,89 @@ var ( metricNameTag = string(iteratorOpts.tagOptions.MetricName()) ) -func TestFetchCompressedReturnsPreloadedData(t *testing.T) { - ctrl := xtest.NewController(t) - defer ctrl.Finish() - +func TestFetchCompressed(t *testing.T) { const ( metricsName = "preloaded" predefinedSeriesCount = 10 ) - query := matcherQuery(t, metricNameTag, metricsName) - - metricsTag := ident.NewTags(ident.Tag{ - Name: ident.BytesID(tagOptions.MetricName()), - Value: ident.BytesID(metricsName), - }) - - iters := make([]encoding.SeriesIterator, 0, predefinedSeriesCount) - for i := 0; i < predefinedSeriesCount; i++ { - iters = append(iters, encoding.NewSeriesIterator( - encoding.SeriesIteratorOptions{ - Namespace: ident.StringID("ns"), - Tags: ident.NewTagsIterator(metricsTag), - StartInclusive: xtime.ToUnixNano(query.Start), - EndExclusive: xtime.ToUnixNano(query.End), - }, nil)) + tests := []struct { + name string + queryTagName string + queryTagValue string + expectedCount int + }{ + { + name: "querying by metric name returns preloaded data", + queryTagName: metricNameTag, + queryTagValue: metricsName, + expectedCount: predefinedSeriesCount, + }, + { + name: "querying without metric name just by other tag returns preloaded data", + queryTagName: "tag1", + queryTagValue: "test2", + expectedCount: 4, + }, } - seriesIterators := encoding.NewMockSeriesIterators(ctrl) - seriesIterators.EXPECT().Len().Return(predefinedSeriesCount).MinTimes(1) - seriesIterators.EXPECT().Iters().Return(iters).Times(1) - seriesIterators.EXPECT().Close() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := xtest.NewController(t) + defer ctrl.Finish() + + query := matcherQuery(t, tt.queryTagName, tt.queryTagValue) - seriesLoader := &testSeriesLoadHandler{seriesIterators} + metricsTag := ident.NewTags(ident.Tag{ + Name: ident.BytesID(tagOptions.MetricName()), + Value: ident.BytesID(metricsName), + }, + ident.Tag{ + Name: ident.BytesID("tag1"), + Value: ident.BytesID("test"), + }, + ) + metricsTag2 := ident.NewTags(ident.Tag{ + Name: ident.BytesID(tagOptions.MetricName()), + Value: ident.BytesID(metricsName), + }, + ident.Tag{ + Name: ident.BytesID("tag1"), + Value: ident.BytesID("test2"), + }, + ) + + iters := make([]encoding.SeriesIterator, 0, predefinedSeriesCount) + for i := 0; i < predefinedSeriesCount; i++ { + m := metricsTag + if i > 5 { + m = metricsTag2 + } + iters = append(iters, encoding.NewSeriesIterator( + encoding.SeriesIteratorOptions{ + Namespace: ident.StringID("ns"), + Tags: ident.NewTagsIterator(m), + StartInclusive: xtime.ToUnixNano(query.Start), + EndExclusive: xtime.ToUnixNano(query.End), + }, nil)) + } - querier := &querier{opts: iteratorOpts, handler: seriesLoader} + seriesIterators := encoding.NewMockSeriesIterators(ctrl) + seriesIterators.EXPECT().Len().Return(predefinedSeriesCount).MinTimes(1) + seriesIterators.EXPECT().Iters().Return(iters).Times(1) + seriesIterators.EXPECT().Close() - result, cleanup, err := querier.FetchCompressed(nil, query, nil) - assert.NoError(t, err) - defer cleanup() + seriesLoader := &testSeriesLoadHandler{seriesIterators} - assert.Equal(t, predefinedSeriesCount, result.SeriesIterators.Len()) + querier := &querier{opts: iteratorOpts, handler: seriesLoader} + + result, cleanup, err := querier.FetchCompressed(nil, query, nil) + assert.NoError(t, err) + defer cleanup() + + assert.Equal(t, tt.expectedCount, result.SeriesIterators.Len()) + }) + } } func TestFetchCompressedGeneratesRandomData(t *testing.T) { diff --git a/src/cmd/services/m3comparator/main/series_load_handler.go b/src/cmd/services/m3comparator/main/series_load_handler.go index 8c60f8c495..9a07be8aa8 100644 --- a/src/cmd/services/m3comparator/main/series_load_handler.go +++ b/src/cmd/services/m3comparator/main/series_load_handler.go @@ -89,7 +89,30 @@ func (l *httpSeriesLoadHandler) getSeriesIterators( defer l.RUnlock() logger := l.iterOpts.iOpts.Logger() - seriesMap, found := l.nameIDSeriesMap[name] + var seriesMap idSeriesMap + found := false + if len(name) > 0 { + seriesMap, found = l.nameIDSeriesMap[name] + } else { + seriesMap = idSeriesMap{ + series: make(map[string]parser.Series), + start: time.Unix(1<<63-62135596801, 999999999), + end: time.Unix(0, 0), + } + for _, series := range l.nameIDSeriesMap { + found = true + for k, v := range series.series { + seriesMap.series[k] = v + } + if series.start.Before(seriesMap.start) { + seriesMap.start = series.start + } + if series.end.After(seriesMap.end) { + seriesMap.end = series.end + } + } + } + if !found || len(seriesMap.series) == 0 { return nil, nil } @@ -198,6 +221,7 @@ func (l *httpSeriesLoadHandler) serveHTTP(r *http.Request) error { if r.Method == http.MethodDelete { l.nameIDSeriesMap = make(map[string]idSeriesMap) + logger.Info("clearing all series") return nil } diff --git a/src/cmd/services/m3comparator/main/series_load_handler_test.go b/src/cmd/services/m3comparator/main/series_load_handler_test.go index 8f77753882..b1700fa5aa 100644 --- a/src/cmd/services/m3comparator/main/series_load_handler_test.go +++ b/src/cmd/services/m3comparator/main/series_load_handler_test.go @@ -203,27 +203,85 @@ func TestClearData(t *testing.T) { recorder := httptest.NewRecorder() - handler := newSeriesLoadHandler(opts) + handler := newHTTPSeriesLoadHandler(opts) handler.ServeHTTP(recorder, req) assert.Equal(t, http.StatusOK, recorder.Code) - iters, err := handler.getSeriesIterators("series_name") + iters, err := handler.getSeriesIterators("series_name") require.NoError(t, err) require.Equal(t, 1, len(iters.Iters())) - - // Call clear data - req, err = http.NewRequest(http.MethodDelete, "", nil) + + // Call clear data + req, err = http.NewRequest(http.MethodDelete, "", nil) require.NoError(t, err) - handler.ServeHTTP(recorder, req) + handler.ServeHTTP(recorder, req) assert.Equal(t, http.StatusOK, recorder.Code) - iters, err = handler.getSeriesIterators("series_name") + iters, err = handler.getSeriesIterators("series_name") require.NoError(t, err) require.Nil(t, iters) } +const seriesStr2 = ` +[ + { + "start": "2020-03-30T11:39:45Z", + "end": "2020-03-30T11:58:00Z", + "tags": [ + ["__name__", "series_name"], + ["abc", "def"], + ["tag_a", "foo"] + ], + "datapoints": [ + { "val": "7076", "ts": "2020-03-30T11:39:51.288Z" }, + { "val": "7079", "ts": "2020-03-30T11:40:18.886Z" }, + { "val": "7094", "ts": "2020-03-30T11:57:57.478Z" } + ] + }, + { + "start": "2020-03-30T11:39:45Z", + "end": "2020-03-30T11:58:00Z", + "tags": [ + ["__name__", "series_name2"], + ["abc", "def"], + ["tag_a", "foo"] + ], + "datapoints": [ + { "val": "8076", "ts": "2020-03-30T11:39:51.288Z" }, + { "val": "8094", "ts": "2020-03-30T11:57:57.478Z" } + ] + } +]` + +func TestNoNameQuerying(t *testing.T) { + opts := iteratorOptions{ + encoderPool: encoderPool, + iteratorPools: iterPools, + tagOptions: tagOptions, + iOpts: iOpts, + } + + req, err := http.NewRequest(http.MethodPost, "", strings.NewReader(seriesStr2)) + require.NoError(t, err) + + recorder := httptest.NewRecorder() + + handler := newHTTPSeriesLoadHandler(opts) + handler.ServeHTTP(recorder, req) + + assert.Equal(t, http.StatusOK, recorder.Code) + + iters, err := handler.getSeriesIterators("series_name") + require.NoError(t, err) + require.Equal(t, 1, len(iters.Iters())) + + iters, err = handler.getSeriesIterators("") + require.NoError(t, err) + require.Equal(t, 2, len(iters.Iters())) +} + func readTags(it encoding.SeriesIterator) parser.Tags { tagIter := it.Tags() tags := make(parser.Tags, 0, tagIter.Len()) From 699f09ac4c0941130b0ba0c7a24267fa61022535 Mon Sep 17 00:00:00 2001 From: Gediminas Date: Wed, 13 May 2020 10:31:08 +0300 Subject: [PATCH 08/23] promql testdata migrated to m3 stack --- scripts/comparator/run.sh | 2 + src/query/test/m3comparator_client.go | 53 + src/query/test/m3query_client.go | 49 + src/query/test/parser/ast.go | 434 +++ src/query/test/parser/functions.go | 277 ++ src/query/test/parser/generated_parser.y | 709 +++++ src/query/test/parser/generated_parser.y.go | 1638 +++++++++++ src/query/test/parser/lex.go | 814 ++++++ src/query/test/parser/lex_test.go | 737 +++++ src/query/test/parser/parse.go | 709 +++++ src/query/test/parser/parse_test.go | 2769 +++++++++++++++++++ src/query/test/parser/printer.go | 175 ++ src/query/test/parser/printer_test.go | 105 + src/query/test/parser/value.go | 45 + src/query/test/promql_test.go | 35 + src/query/test/test.go | 555 ++++ src/query/test/value.go | 306 ++ 17 files changed, 9412 insertions(+) create mode 100644 src/query/test/m3comparator_client.go create mode 100644 src/query/test/m3query_client.go create mode 100644 src/query/test/parser/ast.go create mode 100644 src/query/test/parser/functions.go create mode 100644 src/query/test/parser/generated_parser.y create mode 100644 src/query/test/parser/generated_parser.y.go create mode 100644 src/query/test/parser/lex.go create mode 100644 src/query/test/parser/lex_test.go create mode 100644 src/query/test/parser/parse.go create mode 100644 src/query/test/parser/parse_test.go create mode 100644 src/query/test/parser/printer.go create mode 100644 src/query/test/parser/printer_test.go create mode 100644 src/query/test/parser/value.go create mode 100644 src/query/test/promql_test.go create mode 100644 src/query/test/test.go create mode 100644 src/query/test/value.go diff --git a/scripts/comparator/run.sh b/scripts/comparator/run.sh index 8c7903cf54..5c8056c95a 100755 --- a/scripts/comparator/run.sh +++ b/scripts/comparator/run.sh @@ -73,3 +73,5 @@ $comparator -input=$QUERY_FILE \ -e=$END \ -comparator=$COMPARATOR_WRITE \ -regressionDir=$REGRESSION_DIR + +go test -v -timeout 30s -count=1 github.com/m3db/m3/src/query/test/ \ No newline at end of file diff --git a/src/query/test/m3comparator_client.go b/src/query/test/m3comparator_client.go new file mode 100644 index 0000000000..226f79a0ba --- /dev/null +++ b/src/query/test/m3comparator_client.go @@ -0,0 +1,53 @@ +package test + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" +) + +type m3comparatorClient struct { + host string + port int +} + +func newM3ComparatorClient(host string, port int) *m3comparatorClient { + return &m3comparatorClient{ + host: host, + port: port, + } +} + +func (c *m3comparatorClient) clear() error { + comparatorURL := fmt.Sprintf("http://%s:%d", c.host, c.port) + req, err := http.NewRequest(http.MethodDelete, comparatorURL, nil) + if err != nil { + return err + } + + _, err = http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("could not clear data on m3comparator: %+v", err) + } + return nil +} + +func (c *m3comparatorClient) load(data []byte) error { + comparatorURL := fmt.Sprintf("http://%s:%d", c.host, c.port) + resp, err := http.Post(comparatorURL, "application/json", bytes.NewReader(data)) + if err != nil { + return fmt.Errorf("got error loading data to comparator %v", err) + } + + if resp.StatusCode/200 != 1 { + var bodyString string + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err == nil { + bodyString = string(bodyBytes) + } + return fmt.Errorf("load status code %d. Response: %s", resp.StatusCode, bodyString) + } + + return nil +} diff --git a/src/query/test/m3query_client.go b/src/query/test/m3query_client.go new file mode 100644 index 0000000000..a81a464c9d --- /dev/null +++ b/src/query/test/m3query_client.go @@ -0,0 +1,49 @@ +package test + +import ( + "fmt" + "io/ioutil" + "net/http" + "strconv" + "time" + + "github.com/pkg/errors" +) + +type m3queryClient struct { + host string + port int +} + +func newM3QueryClient(host string, port int) *m3queryClient { + return &m3queryClient{ + host: host, + port: port, + } +} + +func (c *m3queryClient) query(expr string, t time.Time) ([]byte, error) { + url := fmt.Sprintf("http://%s:%d/api/v1/query", c.host, c.port) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + q := req.URL.Query() + q.Add("query", expr) + q.Add("time", strconv.FormatInt(t.Unix(), 10)) + req.URL.RawQuery = q.Encode() + fmt.Printf("Requesting m3query URL: %+v\n", req.URL) + resp, err := http.DefaultClient.Do(req) + + if err != nil { + return nil, errors.Wrapf(err, "error evaluating query %s", expr) + } + + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("invalid status %+v received sending query: %+v", resp.StatusCode, req) + } + + return ioutil.ReadAll(resp.Body) +} diff --git a/src/query/test/parser/ast.go b/src/query/test/parser/ast.go new file mode 100644 index 0000000000..cd7c91c9f5 --- /dev/null +++ b/src/query/test/parser/ast.go @@ -0,0 +1,434 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +import ( + "context" + "time" + + "github.com/pkg/errors" + + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/storage" +) + +// Node is a generic interface for all nodes in an AST. +// +// Whenever numerous nodes are listed such as in a switch-case statement +// or a chain of function definitions (e.g. String(), expr(), etc.) convention is +// to list them as follows: +// +// - Statements +// - statement types (alphabetical) +// - ... +// - Expressions +// - expression types (alphabetical) +// - ... +// +type Node interface { + // String representation of the node that returns the given node when parsed + // as part of a valid query. + String() string + + // PositionRange returns the position of the AST Node in the query string. + PositionRange() PositionRange +} + +// Statement is a generic interface for all statements. +type Statement interface { + Node + + // stmt ensures that no other type accidentally implements the interface + // nolint:unused + stmt() +} + +// EvalStmt holds an expression and information on the range it should +// be evaluated on. +type EvalStmt struct { + Expr Expr // Expression to be evaluated. + + // The time boundaries for the evaluation. If Start equals End an instant + // is evaluated. + Start, End time.Time + // Time between two evaluated instants for the range [Start:End]. + Interval time.Duration +} + +func (*EvalStmt) stmt() {} + +// Expr is a generic interface for all expression types. +type Expr interface { + Node + + // Type returns the type the expression evaluates to. It does not perform + // in-depth checks as this is done at parsing-time. + Type() ValueType + // expr ensures that no other types accidentally implement the interface. + expr() +} + +// Expressions is a list of expression nodes that implements Node. +type Expressions []Expr + +// AggregateExpr represents an aggregation operation on a Vector. +type AggregateExpr struct { + Op ItemType // The used aggregation operation. + Expr Expr // The Vector expression over which is aggregated. + Param Expr // Parameter used by some aggregators. + Grouping []string // The labels by which to group the Vector. + Without bool // Whether to drop the given labels rather than keep them. + PosRange PositionRange +} + +// BinaryExpr represents a binary expression between two child expressions. +type BinaryExpr struct { + Op ItemType // The operation of the expression. + LHS, RHS Expr // The operands on the respective sides of the operator. + + // The matching behavior for the operation if both operands are Vectors. + // If they are not this field is nil. + VectorMatching *VectorMatching + + // If a comparison operator, return 0/1 rather than filtering. + ReturnBool bool +} + +// Call represents a function call. +type Call struct { + Func *Function // The function that was called. + Args Expressions // Arguments used in the call. + + PosRange PositionRange +} + +// MatrixSelector represents a Matrix selection. +type MatrixSelector struct { + // It is safe to assume that this is an VectorSelector + // if the parser hasn't returned an error. + VectorSelector Expr + Range time.Duration + + EndPos Pos +} + +// SubqueryExpr represents a subquery. +type SubqueryExpr struct { + Expr Expr + Range time.Duration + Offset time.Duration + Step time.Duration + + EndPos Pos +} + +// NumberLiteral represents a number. +type NumberLiteral struct { + Val float64 + + PosRange PositionRange +} + +// ParenExpr wraps an expression so it cannot be disassembled as a consequence +// of operator precedence. +type ParenExpr struct { + Expr Expr + PosRange PositionRange +} + +// StringLiteral represents a string. +type StringLiteral struct { + Val string + PosRange PositionRange +} + +// UnaryExpr represents a unary operation on another expression. +// Currently unary operations are only supported for Scalars. +type UnaryExpr struct { + Op ItemType + Expr Expr + + StartPos Pos +} + +// VectorSelector represents a Vector selection. +type VectorSelector struct { + Name string + Offset time.Duration + LabelMatchers []*labels.Matcher + + // The unexpanded seriesSet populated at query preparation time. + UnexpandedSeriesSet storage.SeriesSet + Series []storage.Series + + PosRange PositionRange +} + +// TestStmt is an internal helper statement that allows execution +// of an arbitrary function during handling. It is used to test the Engine. +type TestStmt func(context.Context) error + +func (TestStmt) String() string { return "test statement" } +func (TestStmt) stmt() {} + +func (TestStmt) PositionRange() PositionRange { + return PositionRange{ + Start: -1, + End: -1, + } +} +func (e *AggregateExpr) Type() ValueType { return ValueTypeVector } +func (e *Call) Type() ValueType { return e.Func.ReturnType } +func (e *MatrixSelector) Type() ValueType { return ValueTypeMatrix } +func (e *SubqueryExpr) Type() ValueType { return ValueTypeMatrix } +func (e *NumberLiteral) Type() ValueType { return ValueTypeScalar } +func (e *ParenExpr) Type() ValueType { return e.Expr.Type() } +func (e *StringLiteral) Type() ValueType { return ValueTypeString } +func (e *UnaryExpr) Type() ValueType { return e.Expr.Type() } +func (e *VectorSelector) Type() ValueType { return ValueTypeVector } +func (e *BinaryExpr) Type() ValueType { + if e.LHS.Type() == ValueTypeScalar && e.RHS.Type() == ValueTypeScalar { + return ValueTypeScalar + } + return ValueTypeVector +} + +func (*AggregateExpr) expr() {} +func (*BinaryExpr) expr() {} +func (*Call) expr() {} +func (*MatrixSelector) expr() {} +func (*SubqueryExpr) expr() {} +func (*NumberLiteral) expr() {} +func (*ParenExpr) expr() {} +func (*StringLiteral) expr() {} +func (*UnaryExpr) expr() {} +func (*VectorSelector) expr() {} + +// VectorMatchCardinality describes the cardinality relationship +// of two Vectors in a binary operation. +type VectorMatchCardinality int + +const ( + CardOneToOne VectorMatchCardinality = iota + CardManyToOne + CardOneToMany + CardManyToMany +) + +func (vmc VectorMatchCardinality) String() string { + switch vmc { + case CardOneToOne: + return "one-to-one" + case CardManyToOne: + return "many-to-one" + case CardOneToMany: + return "one-to-many" + case CardManyToMany: + return "many-to-many" + } + panic("promql.VectorMatchCardinality.String: unknown match cardinality") +} + +// VectorMatching describes how elements from two Vectors in a binary +// operation are supposed to be matched. +type VectorMatching struct { + // The cardinality of the two Vectors. + Card VectorMatchCardinality + // MatchingLabels contains the labels which define equality of a pair of + // elements from the Vectors. + MatchingLabels []string + // On includes the given label names from matching, + // rather than excluding them. + On bool + // Include contains additional labels that should be included in + // the result from the side with the lower cardinality. + Include []string +} + +// Visitor allows visiting a Node and its child nodes. The Visit method is +// invoked for each node with the path leading to the node provided additionally. +// If the result visitor w is not nil and no error, Walk visits each of the children +// of node with the visitor w, followed by a call of w.Visit(nil, nil). +type Visitor interface { + Visit(node Node, path []Node) (w Visitor, err error) +} + +// Walk traverses an AST in depth-first order: It starts by calling +// v.Visit(node, path); node must not be nil. If the visitor w returned by +// v.Visit(node, path) is not nil and the visitor returns no error, Walk is +// invoked recursively with visitor w for each of the non-nil children of node, +// followed by a call of w.Visit(nil), returning an error +// As the tree is descended the path of previous nodes is provided. +func Walk(v Visitor, node Node, path []Node) error { + var err error + if v, err = v.Visit(node, path); v == nil || err != nil { + return err + } + path = append(path, node) + + for _, e := range Children(node) { + if err := Walk(v, e, path); err != nil { + return err + } + } + + _, err = v.Visit(nil, nil) + return err +} + +type inspector func(Node, []Node) error + +func (f inspector) Visit(node Node, path []Node) (Visitor, error) { + if err := f(node, path); err != nil { + return nil, err + } + + return f, nil +} + +// Inspect traverses an AST in depth-first order: It starts by calling +// f(node, path); node must not be nil. If f returns a nil error, Inspect invokes f +// for all the non-nil children of node, recursively. +func Inspect(node Node, f inspector) { + //nolint: errcheck + Walk(inspector(f), node, nil) +} + +// Children returns a list of all child nodes of a syntax tree node. +func Children(node Node) []Node { + // For some reasons these switches have significantly better performance than interfaces + switch n := node.(type) { + case *EvalStmt: + return []Node{n.Expr} + case Expressions: + // golang cannot convert slices of interfaces + ret := make([]Node, len(n)) + for i, e := range n { + ret[i] = e + } + return ret + case *AggregateExpr: + // While this does not look nice, it should avoid unnecessary allocations + // caused by slice resizing + if n.Expr == nil && n.Param == nil { + return nil + } else if n.Expr == nil { + return []Node{n.Param} + } else if n.Param == nil { + return []Node{n.Expr} + } else { + return []Node{n.Expr, n.Param} + } + case *BinaryExpr: + return []Node{n.LHS, n.RHS} + case *Call: + // golang cannot convert slices of interfaces + ret := make([]Node, len(n.Args)) + for i, e := range n.Args { + ret[i] = e + } + return ret + case *SubqueryExpr: + return []Node{n.Expr} + case *ParenExpr: + return []Node{n.Expr} + case *UnaryExpr: + return []Node{n.Expr} + case *MatrixSelector: + return []Node{n.VectorSelector} + case *NumberLiteral, *StringLiteral, *VectorSelector: + // nothing to do + return []Node{} + default: + panic(errors.Errorf("promql.Children: unhandled node type %T", node)) + } +} + +// PositionRange describes a position in the input string of the parser. +type PositionRange struct { + Start Pos + End Pos +} + +// mergeRanges is a helper function to merge the PositionRanges of two Nodes. +// Note that the arguments must be in the same order as they +// occur in the input string. +func mergeRanges(first Node, last Node) PositionRange { + return PositionRange{ + Start: first.PositionRange().Start, + End: last.PositionRange().End, + } +} + +// Item implements the Node interface. +// This makes it possible to call mergeRanges on them. +func (i *Item) PositionRange() PositionRange { + return PositionRange{ + Start: i.Pos, + End: i.Pos + Pos(len(i.Val)), + } +} + +func (e *AggregateExpr) PositionRange() PositionRange { + return e.PosRange +} +func (e *BinaryExpr) PositionRange() PositionRange { + return mergeRanges(e.LHS, e.RHS) +} +func (e *Call) PositionRange() PositionRange { + return e.PosRange +} +func (e *EvalStmt) PositionRange() PositionRange { + return e.Expr.PositionRange() +} +func (e Expressions) PositionRange() PositionRange { + if len(e) == 0 { + // Position undefined. + return PositionRange{ + Start: -1, + End: -1, + } + } + return mergeRanges(e[0], e[len(e)-1]) +} +func (e *MatrixSelector) PositionRange() PositionRange { + return PositionRange{ + Start: e.VectorSelector.PositionRange().Start, + End: e.EndPos, + } +} +func (e *SubqueryExpr) PositionRange() PositionRange { + return PositionRange{ + Start: e.Expr.PositionRange().Start, + End: e.EndPos, + } +} +func (e *NumberLiteral) PositionRange() PositionRange { + return e.PosRange +} +func (e *ParenExpr) PositionRange() PositionRange { + return e.PosRange +} +func (e *StringLiteral) PositionRange() PositionRange { + return e.PosRange +} +func (e *UnaryExpr) PositionRange() PositionRange { + return PositionRange{ + Start: e.StartPos, + End: e.Expr.PositionRange().End, + } +} +func (e *VectorSelector) PositionRange() PositionRange { + return e.PosRange +} diff --git a/src/query/test/parser/functions.go b/src/query/test/parser/functions.go new file mode 100644 index 0000000000..4516829e55 --- /dev/null +++ b/src/query/test/parser/functions.go @@ -0,0 +1,277 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parser + +// Function represents a function of the expression language and is +// used by function nodes. +type Function struct { + Name string + ArgTypes []ValueType + Variadic int + ReturnType ValueType +} + +// Functions is a list of all functions supported by PromQL, including their types. +var Functions = map[string]*Function{ + "abs": { + Name: "abs", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "absent": { + Name: "absent", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "absent_over_time": { + Name: "absent_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "avg_over_time": { + Name: "avg_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "ceil": { + Name: "ceil", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "changes": { + Name: "changes", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "clamp_max": { + Name: "clamp_max", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeScalar}, + ReturnType: ValueTypeVector, + }, + "clamp_min": { + Name: "clamp_min", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeScalar}, + ReturnType: ValueTypeVector, + }, + "count_over_time": { + Name: "count_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "days_in_month": { + Name: "days_in_month", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "day_of_month": { + Name: "day_of_month", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "day_of_week": { + Name: "day_of_week", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "delta": { + Name: "delta", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "deriv": { + Name: "deriv", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "exp": { + Name: "exp", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "floor": { + Name: "floor", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "histogram_quantile": { + Name: "histogram_quantile", + ArgTypes: []ValueType{ValueTypeScalar, ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "holt_winters": { + Name: "holt_winters", + ArgTypes: []ValueType{ValueTypeMatrix, ValueTypeScalar, ValueTypeScalar}, + ReturnType: ValueTypeVector, + }, + "hour": { + Name: "hour", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "idelta": { + Name: "idelta", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "increase": { + Name: "increase", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "irate": { + Name: "irate", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "label_replace": { + Name: "label_replace", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeString, ValueTypeString, ValueTypeString, ValueTypeString}, + ReturnType: ValueTypeVector, + }, + "label_join": { + Name: "label_join", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeString, ValueTypeString, ValueTypeString}, + Variadic: -1, + ReturnType: ValueTypeVector, + }, + "ln": { + Name: "ln", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "log10": { + Name: "log10", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "log2": { + Name: "log2", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "max_over_time": { + Name: "max_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "min_over_time": { + Name: "min_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "minute": { + Name: "minute", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "month": { + Name: "month", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "predict_linear": { + Name: "predict_linear", + ArgTypes: []ValueType{ValueTypeMatrix, ValueTypeScalar}, + ReturnType: ValueTypeVector, + }, + "quantile_over_time": { + Name: "quantile_over_time", + ArgTypes: []ValueType{ValueTypeScalar, ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "rate": { + Name: "rate", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "resets": { + Name: "resets", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "round": { + Name: "round", + ArgTypes: []ValueType{ValueTypeVector, ValueTypeScalar}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, + "scalar": { + Name: "scalar", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeScalar, + }, + "sort": { + Name: "sort", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "sort_desc": { + Name: "sort_desc", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "sqrt": { + Name: "sqrt", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "stddev_over_time": { + Name: "stddev_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "stdvar_over_time": { + Name: "stdvar_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "sum_over_time": { + Name: "sum_over_time", + ArgTypes: []ValueType{ValueTypeMatrix}, + ReturnType: ValueTypeVector, + }, + "time": { + Name: "time", + ArgTypes: []ValueType{}, + ReturnType: ValueTypeScalar, + }, + "timestamp": { + Name: "timestamp", + ArgTypes: []ValueType{ValueTypeVector}, + ReturnType: ValueTypeVector, + }, + "vector": { + Name: "vector", + ArgTypes: []ValueType{ValueTypeScalar}, + ReturnType: ValueTypeVector, + }, + "year": { + Name: "year", + ArgTypes: []ValueType{ValueTypeVector}, + Variadic: 1, + ReturnType: ValueTypeVector, + }, +} + +// getFunction returns a predefined Function object for the given name. +func getFunction(name string) (*Function, bool) { + function, ok := Functions[name] + return function, ok +} diff --git a/src/query/test/parser/generated_parser.y b/src/query/test/parser/generated_parser.y new file mode 100644 index 0000000000..f7c31c1aa5 --- /dev/null +++ b/src/query/test/parser/generated_parser.y @@ -0,0 +1,709 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +%{ +package parser + +import ( + "math" + "sort" + "strconv" + "time" + + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/value" +) +%} + +%union { + node Node + item Item + matchers []*labels.Matcher + matcher *labels.Matcher + label labels.Label + labels labels.Labels + strings []string + series []SequenceValue + uint uint64 + float float64 + duration time.Duration +} + + +%token +ASSIGN +BLANK +COLON +COMMA +COMMENT +DURATION +EOF +ERROR +IDENTIFIER +LEFT_BRACE +LEFT_BRACKET +LEFT_PAREN +METRIC_IDENTIFIER +NUMBER +RIGHT_BRACE +RIGHT_BRACKET +RIGHT_PAREN +SEMICOLON +SPACE +STRING +TIMES + +// Operators. +%token operatorsStart +%token +ADD +DIV +EQL +EQL_REGEX +GTE +GTR +LAND +LOR +LSS +LTE +LUNLESS +MOD +MUL +NEQ +NEQ_REGEX +POW +SUB +%token operatorsEnd + +// Aggregators. +%token aggregatorsStart +%token +AVG +BOTTOMK +COUNT +COUNT_VALUES +MAX +MIN +QUANTILE +STDDEV +STDVAR +SUM +TOPK +%token aggregatorsEnd + +// Keywords. +%token keywordsStart +%token +BOOL +BY +GROUP_LEFT +GROUP_RIGHT +IGNORING +OFFSET +ON +WITHOUT +%token keywordsEnd + + +// Start symbols for the generated parser. +%token startSymbolsStart +%token +START_METRIC +START_SERIES_DESCRIPTION +START_EXPRESSION +START_METRIC_SELECTOR +%token startSymbolsEnd + + +// Type definitions for grammar rules. +%type label_match_list +%type label_matcher + +%type aggregate_op grouping_label match_op maybe_label metric_identifier unary_op + +%type label_set label_set_list metric +%type