From a5664b96716594fa1b37599f9171437c101eabcd Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 10 Dec 2024 11:06:29 -0600 Subject: [PATCH 1/7] Move the hotQueries logic out of the rendering Signed-off-by: Florent Poinsard --- go/summarize/markdown.go | 37 +++----- go/summarize/summarize-keys.go | 22 +++-- go/summarize/summarize-keys_test.go | 3 + go/summarize/summarize-planalyze.go | 3 + go/summarize/summarize.go | 40 ++++++++ go/summarize/summary.go | 30 +++++- .../bigger_slow_log_avg-rows-examined.md | 22 ++--- .../bigger_slow_log_total-rows-examined.md | 10 +- .../bigger_slow_log_usage-count.md | 26 +++--- .../bigger_slow_query_plan_report.md | 93 ++++++++----------- 10 files changed, 173 insertions(+), 113 deletions(-) diff --git a/go/summarize/markdown.go b/go/summarize/markdown.go index 56bcfd9..927f4ae 100644 --- a/go/summarize/markdown.go +++ b/go/summarize/markdown.go @@ -27,29 +27,15 @@ import ( humanize "github.com/dustin/go-humanize" - "github.com/vitessio/vt/go/keys" "github.com/vitessio/vt/go/markdown" "github.com/vitessio/vt/go/planalyze" ) -func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, metricReader getMetric) { +func renderHotQueries(md *markdown.MarkDown, queries []HotQueryResult) { if len(queries) == 0 { return } - hasTime := false - // Sort the queries in descending order of hotness - sort.Slice(queries, func(i, j int) bool { - if queries[i].QueryTime != 0 { - hasTime = true - } - return metricReader(queries[i]) > metricReader(queries[j]) - }) - - if !hasTime { - return - } - md.PrintHeader("Top Queries", 2) // Prepare table headers and rows @@ -58,13 +44,12 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, for i, query := range queries { queryID := fmt.Sprintf("Q%d", i+1) - avgQueryTime := query.QueryTime / float64(query.UsageCount) rows = append(rows, []string{ queryID, - humanize.Comma(int64(query.UsageCount)), - fmt.Sprintf("%.2f", query.QueryTime), - fmt.Sprintf("%.2f", avgQueryTime), - humanize.Comma(int64(query.RowsExamined)), + humanize.Comma(int64(query.QueryAnalysisResult.UsageCount)), + fmt.Sprintf("%.2f", query.QueryAnalysisResult.QueryTime), + fmt.Sprintf("%.2f", query.AvgQueryTime), + humanize.Comma(int64(query.QueryAnalysisResult.RowsExamined)), }) } @@ -77,7 +62,7 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, queryID := fmt.Sprintf("Q%d", i+1) md.PrintHeader(queryID, 4) md.Println("```sql") - md.Println(query.QueryStructure) + md.Println(query.QueryAnalysisResult.QueryStructure) md.Println("```") md.NewLine() } @@ -258,10 +243,13 @@ func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error { func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery, title string) error { for i, query := range queries { if i == 0 { - md.Printf("# %s Queries\n\n", title) + md.PrintHeader(fmt.Sprintf("%s Queries\n\n", title), 3) } - md.Printf("## Query\n\n```sql\n%s\n```\n\n", query.QueryStructure) - md.Println("## Plan\n\n```json") + md.PrintHeader("Query", 4) + md.Printf("```sql\n%s\n```\n\n", query.QueryStructure) + + md.PrintHeader("Plan", 4) + md.Println("```json") // Indent the JSON output. If we don't do this, the json will be indented all wrong var buf bytes.Buffer @@ -274,6 +262,7 @@ func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery, } md.NewLine() md.Println("```") + md.Println("---") md.NewLine() } return nil diff --git a/go/summarize/summarize-keys.go b/go/summarize/summarize-keys.go index 80a5add..6bb71d8 100644 --- a/go/summarize/summarize-keys.go +++ b/go/summarize/summarize-keys.go @@ -194,8 +194,8 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error { // First pass: collect all graphData and count occurrences for _, query := range queries.Queries { + summary.addQueryResult(query) gatherTableInfo(query, tableSummaries, tableUsageWriteCounts, tableUsageReadCounts) - checkQueryForHotness(&summary.hotQueries, query, summary.hotQueryFn) } // Second pass: calculate percentages @@ -274,15 +274,25 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error { return nil } -func checkQueryForHotness(hotQueries *[]keys.QueryAnalysisResult, query keys.QueryAnalysisResult, metricReader getMetric) { +func checkQueryForHotness(hotQueries *[]HotQueryResult, query QueryResult, metricReader getMetric) { // todo: we should be able to choose different metrics for hotness - e.g. total time spent on query, number of rows examined, etc. + newHotQueryFn := func() HotQueryResult { + return HotQueryResult{ + QueryResult: QueryResult{ + QueryAnalysisResult: query.QueryAnalysisResult, + PlanAnalysis: query.PlanAnalysis, + }, + AvgQueryTime: query.QueryAnalysisResult.QueryTime / float64(query.QueryAnalysisResult.UsageCount), + } + } + switch { case len(*hotQueries) < HotQueryCount: // If we have not yet reached the limit, add the query - *hotQueries = append(*hotQueries, query) - case metricReader(query) > metricReader((*hotQueries)[0]): + *hotQueries = append(*hotQueries, newHotQueryFn()) + case metricReader(query.QueryAnalysisResult) > metricReader((*hotQueries)[0].QueryAnalysisResult): // If the current query has more usage than the least used hot query, replace it - (*hotQueries)[0] = query + (*hotQueries)[0] = newHotQueryFn() default: // If the current query is not hot enough, just return return @@ -291,7 +301,7 @@ func checkQueryForHotness(hotQueries *[]keys.QueryAnalysisResult, query keys.Que // Sort the hot queries by query time so that the least used query is always at the front sort.Slice(*hotQueries, func(i, j int) bool { - return metricReader((*hotQueries)[i]) < metricReader((*hotQueries)[j]) + return metricReader((*hotQueries)[i].QueryAnalysisResult) < metricReader((*hotQueries)[j].QueryAnalysisResult) }) } diff --git a/go/summarize/summarize-keys_test.go b/go/summarize/summarize-keys_test.go index ba6c0df..1a18ec5 100644 --- a/go/summarize/summarize-keys_test.go +++ b/go/summarize/summarize-keys_test.go @@ -118,6 +118,9 @@ func TestSummarizeKeysWithHotnessFile(t *testing.T) { err = fn(s) require.NoError(t, err) + err = compileSummary(s) + require.NoError(t, err) + err = s.PrintMarkdown(sb, now) require.NoError(t, err) diff --git a/go/summarize/summarize-planalyze.go b/go/summarize/summarize-planalyze.go index ddda77c..0e8f26f 100644 --- a/go/summarize/summarize-planalyze.go +++ b/go/summarize/summarize-planalyze.go @@ -28,6 +28,9 @@ func summarizePlanAnalyze(s *Summary, data planalyze.Output) (err error) { Unplannable: len(data.Unplannable), } + s.addPlanResult(data.SimpleRouted) + s.addPlanResult(data.Complex) + s.planAnalysis.simpleRouted = append(s.planAnalysis.simpleRouted, data.SimpleRouted...) s.planAnalysis.complex = append(s.planAnalysis.complex, data.Complex...) return nil diff --git a/go/summarize/summarize.go b/go/summarize/summarize.go index 6994f4f..0585eaa 100644 --- a/go/summarize/summarize.go +++ b/go/summarize/summarize.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "sort" "strings" "time" @@ -115,6 +116,12 @@ func printSummary(hotMetric string, workers []summaryWorker) (*Summary, error) { return nil, err } } + + err = compileSummary(s) + if err != nil { + return nil, err + } + err = s.PrintMarkdown(os.Stdout, time.Now()) if err != nil { return nil, err @@ -122,6 +129,39 @@ func printSummary(hotMetric string, workers []summaryWorker) (*Summary, error) { return s, nil } +func compileSummary(s *Summary) error { + if err := compileHotQueries(s); err != nil { + return err + } + return nil +} + +func compileHotQueries(s *Summary) error { + for _, result := range s.queries { + checkQueryForHotness(&s.hotQueries, result, s.hotQueryFn) + } + var hasTime bool + sort.Slice(s.hotQueries, func(i, j int) bool { + if s.hotQueries[i].QueryAnalysisResult.QueryTime != 0 { + hasTime = true + } + fnI := s.hotQueryFn(s.hotQueries[i].QueryAnalysisResult) + fnJ := s.hotQueryFn(s.hotQueries[j].QueryAnalysisResult) + + // if the two metrics are equal, sort them by alphabetical order + if fnI == fnJ { + return s.hotQueries[i].QueryAnalysisResult.QueryStructure > s.hotQueries[j].QueryAnalysisResult.QueryStructure + } + return fnI > fnJ + }) + + // If we did not record any time, there is no hotness to record, so removing the field so it does not get rendered. + if !hasTime { + s.hotQueries = nil + } + return nil +} + func checkTraceConditions(traces []traceSummary, workers []summaryWorker, hotMetric string) error { if len(workers) > 0 { return errors.New("trace files cannot be mixed with other file types") diff --git a/go/summarize/summary.go b/go/summarize/summary.go index 32f62ca..41d3c18 100644 --- a/go/summarize/summary.go +++ b/go/summarize/summary.go @@ -36,12 +36,23 @@ type ( failures []FailuresSummary transactions []TransactionSummary planAnalysis PlanAnalysis - hotQueries []keys.QueryAnalysisResult + hotQueries []HotQueryResult hotQueryFn getMetric analyzedFiles []string queryGraph queryGraph joins []joinDetails hasRowCount bool + queries map[string]QueryResult + } + + QueryResult struct { + QueryAnalysisResult keys.QueryAnalysisResult + PlanAnalysis planalyze.AnalyzedQuery + } + + HotQueryResult struct { + QueryResult + AvgQueryTime float64 } TableSummary struct { @@ -90,10 +101,25 @@ func NewSummary(hotMetric string) (*Summary, error) { return &Summary{ queryGraph: make(queryGraph), + queries: make(map[string]QueryResult), hotQueryFn: hotness, }, nil } +func (s *Summary) addQueryResult(qr keys.QueryAnalysisResult) { + val := s.queries[qr.QueryStructure] + val.QueryAnalysisResult = qr + s.queries[qr.QueryStructure] = val +} + +func (s *Summary) addPlanResult(p []planalyze.AnalyzedQuery) { + for _, query := range p { + val := s.queries[query.QueryStructure] + val.PlanAnalysis = query + s.queries[query.QueryStructure] = val + } +} + func (s *Summary) PrintMarkdown(out io.Writer, now time.Time) error { md := &markdown.MarkDown{} filePlural := "" @@ -114,7 +140,7 @@ func (s *Summary) PrintMarkdown(out io.Writer, now time.Time) error { if err != nil { return err } - renderHotQueries(md, s.hotQueries, s.hotQueryFn) + renderHotQueries(md, s.hotQueries) renderTableUsage(md, s.tables, s.hasRowCount) renderTablesJoined(md, s) renderTransactions(md, s.transactions) diff --git a/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md b/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md index 1b717e5..0032ec0 100644 --- a/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md +++ b/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md @@ -6,12 +6,12 @@ ## Top Queries |Query ID|Usage Count|Total Query Time (ms)|Avg Query Time (ms)|Total Rows Examined| |---|---|---|---|---| -|Q1|2|0.40|0.20|20,000| -|Q2|3|0.61|0.20|30,000| -|Q3|1|0.22|0.22|8,000| -|Q4|2|0.37|0.19|16,000| +|Q1|3|0.61|0.20|30,000| +|Q2|2|0.40|0.20|20,000| +|Q3|2|0.49|0.25|16,000| +|Q4|1|0.22|0.22|8,000| |Q5|2|0.37|0.19|16,000| -|Q6|2|0.49|0.25|16,000| +|Q6|2|0.37|0.19|16,000| |Q7|2|0.31|0.16|15,000| |Q8|1|0.20|0.20|6,500| |Q9|3|0.58|0.19|17,000| @@ -20,32 +20,32 @@ ### Query Details #### Q1 ```sql -SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ +SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ ``` #### Q2 ```sql -SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ +SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` #### Q3 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` JOIN `reviews` AS `r` ON `u`.`id` = `r`.`user_id` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) AND `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` #### Q4 ```sql -SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` JOIN `reviews` AS `r` ON `u`.`id` = `r`.`user_id` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) AND `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) ``` #### Q5 ```sql -SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` #### Q6 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` #### Q7 diff --git a/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md b/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md index 83d48eb..64de635 100644 --- a/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md +++ b/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md @@ -9,9 +9,9 @@ |Q1|3|0.61|0.20|30,000| |Q2|2|0.40|0.20|20,000| |Q3|3|0.58|0.19|17,000| -|Q4|2|0.37|0.19|16,000| +|Q4|2|0.49|0.25|16,000| |Q5|2|0.37|0.19|16,000| -|Q6|2|0.49|0.25|16,000| +|Q6|2|0.37|0.19|16,000| |Q7|2|0.31|0.16|15,000| |Q8|2|0.34|0.17|8,500| |Q9|1|0.22|0.22|8,000| @@ -35,17 +35,17 @@ SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS #### Q4 ```sql -SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` #### Q5 ```sql -SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` #### Q6 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` #### Q7 diff --git a/go/testdata/summarize-output/bigger_slow_log_usage-count.md b/go/testdata/summarize-output/bigger_slow_log_usage-count.md index e195fa0..995006c 100644 --- a/go/testdata/summarize-output/bigger_slow_log_usage-count.md +++ b/go/testdata/summarize-output/bigger_slow_log_usage-count.md @@ -8,14 +8,14 @@ |---|---|---|---|---| |Q1|3|0.58|0.19|17,000| |Q2|3|0.61|0.20|30,000| -|Q3|2|0.21|0.11|3,000| -|Q4|2|0.37|0.19|16,000| -|Q5|2|0.31|0.16|15,000| -|Q6|2|0.40|0.20|20,000| +|Q3|2|0.49|0.25|16,000| +|Q4|2|0.33|0.17|6,000| +|Q5|2|0.21|0.11|3,000| +|Q6|2|0.31|0.16|15,000| |Q7|2|0.34|0.17|8,500| -|Q8|2|0.33|0.17|6,000| +|Q8|2|0.40|0.20|20,000| |Q9|2|0.37|0.19|16,000| -|Q10|2|0.49|0.25|16,000| +|Q10|2|0.37|0.19|16,000| ### Query Details #### Q1 @@ -30,22 +30,22 @@ SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` #### Q3 ```sql -SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` #### Q4 ```sql -SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` ``` #### Q5 ```sql -SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ +SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ ``` #### Q6 ```sql -SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ +SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ ``` #### Q7 @@ -55,17 +55,17 @@ SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS #### Q8 ```sql -SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` +SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` #### Q9 ```sql -SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` #### Q10 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` ## Tables diff --git a/go/testdata/summarize-output/bigger_slow_query_plan_report.md b/go/testdata/summarize-output/bigger_slow_query_plan_report.md index 9b03400..a49c418 100644 --- a/go/testdata/summarize-output/bigger_slow_query_plan_report.md +++ b/go/testdata/summarize-output/bigger_slow_query_plan_report.md @@ -13,16 +13,15 @@ |Total|13| -# Simple routed Queries +### Simple routed Queries -## Query +#### Query ```sql SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Route", @@ -36,15 +35,14 @@ SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS "Table": "inventory, products" } ``` +--- -## Query - +#### Query ```sql SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` BETWEEN :1 /* INT64 */ AND :2 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Route", @@ -58,17 +56,17 @@ SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS "Table": "inventory, products" } ``` +--- -# Complex routed Queries +### Complex routed Queries -## Query +#### Query ```sql SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Limit", @@ -89,15 +87,14 @@ SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN ] } ``` +--- -## Query - +#### Query ```sql SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`created_at` BETWEEN :1 /* VARCHAR */ AND :2 /* VARCHAR */ GROUP BY `u`.`id` HAVING sum(`o`.`total_amount`) > :_total_spent /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -170,15 +167,14 @@ SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS ] } ``` +--- -## Query - +#### Query ```sql SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` -## Plan - +#### Plan ```json { "OperatorType": "Aggregate", @@ -317,15 +313,14 @@ SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOI ] } ``` +--- -## Query - +#### Query ```sql SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Limit", @@ -442,15 +437,14 @@ SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `ca ] } ``` +--- -## Query - +#### Query ```sql SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS `s` ON `o`.`id` = `s`.`order_id` WHERE `s`.`shipped_date` IS NULL AND `o`.`created_at` < DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -493,15 +487,14 @@ SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS ] } ``` +--- -## Query - +#### Query ```sql SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` ``` -## Plan - +#### Plan ```json { "OperatorType": "Projection", @@ -570,15 +563,14 @@ SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM ` ] } ``` +--- -## Query - +#### Query ```sql SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` -## Plan - +#### Plan ```json { "OperatorType": "Aggregate", @@ -602,15 +594,14 @@ SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `o ] } ``` +--- -## Query - +#### Query ```sql SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -640,15 +631,14 @@ SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` ] } ``` +--- -## Query - +#### Query ```sql SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -691,15 +681,14 @@ SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON ] } ``` +--- -## Query - +#### Query ```sql SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` JOIN `reviews` AS `r` ON `u`.`id` = `r`.`user_id` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) AND `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) ``` -## Plan - +#### Plan ```json { "OperatorType": "Join", @@ -761,15 +750,14 @@ SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`. ] } ``` +--- -## Query - +#### Query ```sql SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` WHERE `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ week) GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :2 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Limit", @@ -790,4 +778,5 @@ SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN ] } ``` +--- From a986b21bb352585df4dedd75e38855bb47014230 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 10 Dec 2024 11:46:23 -0600 Subject: [PATCH 2/7] Use keys and hot metrics in the planalyze summary test Signed-off-by: Florent Poinsard --- go/summarize/summarize-planalyze_test.go | 16 +- .../bigger_slow_query_plan_report.md | 179 +++++++++++++++++- 2 files changed, 191 insertions(+), 4 deletions(-) diff --git a/go/summarize/summarize-planalyze_test.go b/go/summarize/summarize-planalyze_test.go index ace0721..f6ca648 100644 --- a/go/summarize/summarize-planalyze_test.go +++ b/go/summarize/summarize-planalyze_test.go @@ -27,15 +27,25 @@ import ( ) func TestSummarizePlans(t *testing.T) { - fn, err := readPlanalyzeFile("../testdata/planalyze-output/bigger_slow_query_plan_report.json") + fnPlan, err := readPlanalyzeFile("../testdata/planalyze-output/bigger_slow_query_plan_report.json") require.NoError(t, err) + + fnKeys, err := readKeysFile("../testdata/keys-output/bigger_slow_query_log.json") + require.NoError(t, err) + sb := &strings.Builder{} now := time.Date(2024, time.January, 1, 1, 2, 3, 0, time.UTC) - s, err := NewSummary("") + s, err := NewSummary("usage-count") + require.NoError(t, err) + + err = fnPlan(s) + require.NoError(t, err) + + err = fnKeys(s) require.NoError(t, err) - err = fn(s) + err = compileSummary(s) require.NoError(t, err) err = s.PrintMarkdown(sb, now) diff --git a/go/testdata/summarize-output/bigger_slow_query_plan_report.md b/go/testdata/summarize-output/bigger_slow_query_plan_report.md index a49c418..06dbef8 100644 --- a/go/testdata/summarize-output/bigger_slow_query_plan_report.md +++ b/go/testdata/summarize-output/bigger_slow_query_plan_report.md @@ -1,7 +1,7 @@ # Query Analysis Report **Date of Analysis**: 2024-01-01 01:02:03 -**Analyzed File**: `../testdata/planalyze-output/bigger_slow_query_plan_report.json` +**Analyzed Files**: `../testdata/planalyze-output/bigger_slow_query_plan_report.json`, `../testdata/keys-output/bigger_slow_query_log.json` ## Query Planning Report |Plan Complexity|Count| @@ -780,3 +780,180 @@ SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN ``` --- +## Top Queries +|Query ID|Usage Count|Total Query Time (ms)|Avg Query Time (ms)|Total Rows Examined| +|---|---|---|---|---| +|Q1|3|0.58|0.19|17,000| +|Q2|3|0.61|0.20|30,000| +|Q3|2|0.49|0.25|16,000| +|Q4|2|0.33|0.17|6,000| +|Q5|2|0.21|0.11|3,000| +|Q6|2|0.31|0.16|15,000| +|Q7|2|0.34|0.17|8,500| +|Q8|2|0.40|0.20|20,000| +|Q9|2|0.37|0.19|16,000| +|Q10|2|0.37|0.19|16,000| + +### Query Details +#### Q1 +```sql +SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`created_at` BETWEEN :1 /* VARCHAR */ AND :2 /* VARCHAR */ GROUP BY `u`.`id` HAVING sum(`o`.`total_amount`) > :_total_spent /* INT64 */ +``` + +#### Q2 +```sql +SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ +``` + +#### Q3 +```sql +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +``` + +#### Q4 +```sql +SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` +``` + +#### Q5 +```sql +SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ +``` + +#### Q6 +```sql +SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ +``` + +#### Q7 +```sql +SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS `s` ON `o`.`id` = `s`.`order_id` WHERE `s`.`shipped_date` IS NULL AND `o`.`created_at` < DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) +``` + +#### Q8 +```sql +SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ +``` + +#### Q9 +```sql +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +``` + +#### Q10 +```sql +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +``` + +## Tables +|Table Name|Reads|Writes| +|---|---|---| +|orders|14|0| +|products|10|0| +|users|6|0| +|categories|4|0| +|order_items|4|0| +|reviews|4|0| +|inventory|3|0| +|messages|3|0| +|payments|2|0| +|shipments|2|0| + +### Column Usage +#### Table: `orders` (14 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|created_at|WHERE RANGE|57%| +|id|JOIN|43%| +|user_id|JOIN|43%| + +#### Table: `products` (10 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|id|JOIN|100%| +||GROUP|30%| +|category_id|JOIN|40%| + +#### Table: `users` (6 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|id|JOIN|100%| +||GROUP|50%| + +#### Table: `categories` (4 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|id|JOIN|100%| +||GROUP|100%| + +#### Table: `order_items` (4 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|product_id|JOIN|100%| +|order_id|JOIN|50%| + +#### Table: `reviews` (4 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|product_id|JOIN|75%| +|created_at|WHERE RANGE|50%| +|user_id|JOIN|25%| + +#### Table: `inventory` (3 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|product_id|JOIN|100%| +|stock_level|WHERE RANGE|100%| + +#### Table: `messages` (3 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|sender_id|GROUP|100%| + +#### Table: `payments` (2 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|order_id|JOIN|100%| +|payment_method|GROUP|100%| + +#### Table: `shipments` (2 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|order_id|JOIN|100%| + +## Tables Joined +``` +orders ↔ users (Occurrences: 3) +└─ orders.user_id = users.id + +categories ↔ products (Occurrences: 2) +└─ categories.id = products.category_id + +inventory ↔ products (Occurrences: 2) +└─ inventory.product_id = products.id + +order_items ↔ products (Occurrences: 2) +└─ order_items.product_id = products.id + +products ↔ reviews (Occurrences: 2) +└─ products.id = reviews.product_id + +order_items ↔ orders (Occurrences: 1) +└─ order_items.order_id = orders.id + +orders ↔ payments (Occurrences: 1) +└─ orders.id = payments.order_id + +orders ↔ shipments (Occurrences: 1) +└─ orders.id = shipments.order_id + +reviews ↔ users (Occurrences: 1) +└─ reviews.user_id = users.id + +``` +## Failures +|Error|Count| +|---|---| +|syntax error at position 2|1| +|syntax error at position 14 near 'timestamp'|1| + From c91606f07884a871bbbf98b39d1a338cc7d54059 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Tue, 10 Dec 2024 12:36:47 -0600 Subject: [PATCH 3/7] Combine hot queries with query plan analysis Signed-off-by: Florent Poinsard --- go/summarize/markdown.go | 13 + .../bigger_slow_query_plan_report.md | 583 +++++++++++++++++- 2 files changed, 586 insertions(+), 10 deletions(-) diff --git a/go/summarize/markdown.go b/go/summarize/markdown.go index 927f4ae..c957679 100644 --- a/go/summarize/markdown.go +++ b/go/summarize/markdown.go @@ -59,11 +59,24 @@ func renderHotQueries(md *markdown.MarkDown, queries []HotQueryResult) { // After the table, list the full queries with their IDs md.PrintHeader("Query Details", 3) for i, query := range queries { + hasPlanAnalysis := len(string(query.PlanAnalysis.PlanOutput)) > 0 + queryID := fmt.Sprintf("Q%d", i+1) + if hasPlanAnalysis { + queryID += fmt.Sprintf(" (`%s`)", query.PlanAnalysis.Complexity.String()) + } + md.PrintHeader(queryID, 4) md.Println("```sql") md.Println(query.QueryAnalysisResult.QueryStructure) md.Println("```") + + if hasPlanAnalysis { + md.Println("```json") + md.Println(string(query.PlanAnalysis.PlanOutput)) + md.Println("```") + } + md.NewLine() } } diff --git a/go/testdata/summarize-output/bigger_slow_query_plan_report.md b/go/testdata/summarize-output/bigger_slow_query_plan_report.md index 06dbef8..c7baff1 100644 --- a/go/testdata/summarize-output/bigger_slow_query_plan_report.md +++ b/go/testdata/summarize-output/bigger_slow_query_plan_report.md @@ -795,55 +795,618 @@ SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN |Q10|2|0.37|0.19|16,000| ### Query Details -#### Q1 +#### Q1 (`Complex routed`) ```sql SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`created_at` BETWEEN :1 /* VARCHAR */ AND :2 /* VARCHAR */ GROUP BY `u`.`id` HAVING sum(`o`.`total_amount`) > :_total_spent /* INT64 */ ``` +```json +{ + "OperatorType": "Filter", + "Predicate": "sum(o.total_amount) \u003e :_total_spent", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "any_value(0) AS username, sum(1) AS total_spent", + "GroupBy": "(2|3)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as username", + "sum(o.total_amount) * count(*) as total_spent", + ":3 as id", + ":4 as weight_string(u.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "o_user_id": 1 + }, + "TableName": "orders_users", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(o.total_amount) as total_spent, o.user_id from orders as o where 1 != 1 group by o.user_id", + "Query": "select sum(o.total_amount) as total_spent, o.user_id from orders as o where o.created_at between :1 and :2 group by o.user_id", + "Table": "orders" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select u.username, count(*), u.id, weight_string(u.id) from users as u where 1 != 1 group by u.id, weight_string(u.id)", + "Query": "select u.username, count(*), u.id, weight_string(u.id) from users as u where u.id = :o_user_id group by u.id, weight_string(u.id)", + "Table": "users", + "Values": [ + ":o_user_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } + ] + } +``` -#### Q2 +#### Q2 (`Complex routed`) ```sql SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ ``` +```json +{ + "OperatorType": "Filter", + "Predicate": "count(distinct m.receiver_id) \u003e :_unique_receivers", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "count_distinct(1|3) AS unique_receivers", + "GroupBy": "(0|2)", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id) from messages as m where 1 != 1 group by m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id)", + "OrderBy": "(0|2) ASC, (1|3) ASC", + "Query": "select m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id) from messages as m group by m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id) order by m.sender_id asc, m.receiver_id asc", + "Table": "messages" + } + ] + } + ] + } +``` -#### Q3 +#### Q3 (`Complex routed`) ```sql SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` +```json +{ + "OperatorType": "Filter", + "Predicate": "o.id is null", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "LeftJoin", + "JoinColumnIndexes": "L:0,L:1,R:0", + "JoinVars": { + "u_id": 0 + }, + "TableName": "users_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select u.id, u.username from users as u where 1 != 1", + "Query": "select u.id, u.username from users as u", + "Table": "users" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select o.id from orders as o where 1 != 1", + "Query": "select o.id from orders as o where o.user_id = :u_id", + "Table": "orders" + } + ] + } + ] + } +``` -#### Q4 +#### Q4 (`Complex routed`) ```sql SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` ``` +```json +{ + "OperatorType": "Projection", + "Expressions": [ + ":0 as payment_method", + "sum(o.total_amount) / count(o.total_amount) as avg_order_value" + ], + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum(1) AS avg_order_value, sum_count(2) AS count(o.total_amount)", + "GroupBy": "(0|3)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":3 as payment_method", + "count(*) * sum(o.total_amount) as avg_order_value", + "count(*) * count(o.total_amount) as count(o.total_amount)", + ":4 as weight_string(p.payment_method)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,L:1,L:3", + "JoinVars": { + "p_order_id": 2 + }, + "TableName": "payments_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), p.payment_method, p.order_id, weight_string(p.payment_method) from payments as p where 1 != 1 group by p.payment_method, p.order_id, weight_string(p.payment_method)", + "OrderBy": "(1|3) ASC", + "Query": "select count(*), p.payment_method, p.order_id, weight_string(p.payment_method) from payments as p group by p.payment_method, p.order_id, weight_string(p.payment_method) order by p.payment_method asc", + "Table": "payments" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(o.total_amount) as avg_order_value, count(o.total_amount) from orders as o where 1 != 1 group by .0", + "Query": "select sum(o.total_amount) as avg_order_value, count(o.total_amount) from orders as o where o.id = :p_order_id group by .0", + "Table": "orders", + "Values": [ + ":p_order_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } +``` -#### Q5 +#### Q5 (`Complex routed`) ```sql SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ ``` +```json +{ + "OperatorType": "Limit", + "Count": "_vt_column_1", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select p.`name`, avg(r.rating) as avg_rating from products as p, reviews as r where 1 != 1 group by p.id", + "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci", + "Query": "select p.`name`, avg(r.rating) as avg_rating from products as p, reviews as r where p.id = r.product_id group by p.id order by avg(r.rating) desc limit :1", + "Table": "products, reviews" + } + ] + } +``` -#### Q6 +#### Q6 (`Simple routed`) ```sql SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ ``` +```json +{ + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select p.`name`, i.stock_level from products as p, inventory as i where 1 != 1", + "Query": "select p.`name`, i.stock_level from products as p, inventory as i where i.stock_level \u003c :_i_stock_level and p.id = i.product_id", + "Table": "inventory, products" + } +``` -#### Q7 +#### Q7 (`Complex routed`) ```sql SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS `s` ON `o`.`id` = `s`.`order_id` WHERE `s`.`shipped_date` IS NULL AND `o`.`created_at` < DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) ``` +```json +{ + "OperatorType": "Filter", + "Predicate": "s.shipped_date is null", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "LeftJoin", + "JoinColumnIndexes": "L:0,L:1,R:0", + "JoinVars": { + "o_id": 0 + }, + "TableName": "orders_shipments", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select o.id, o.created_at from orders as o where 1 != 1", + "Query": "select o.id, o.created_at from orders as o where o.created_at \u003c date_sub(now(), interval :1 day)", + "Table": "orders" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select s.shipped_date from shipments as s where 1 != 1", + "Query": "select s.shipped_date from shipments as s where s.order_id = :o_id", + "Table": "shipments" + } + ] + } + ] + } +``` -#### Q8 +#### Q8 (`Complex routed`) ```sql SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` +```json +{ + "OperatorType": "Limit", + "Count": "_vt_column_1", + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "any_value(0) AS name, sum(1) AS total_sales", + "GroupBy": "(2|3)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "sum(oi.price * oi.quantity) * count(*) as total_sales", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "oi_product_id": 1 + }, + "TableName": "order_items_products_categories", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(oi.price * oi.quantity) as total_sales, oi.product_id from order_items as oi where 1 != 1 group by oi.product_id", + "Query": "select sum(oi.price * oi.quantity) as total_sales, oi.product_id from order_items as oi group by oi.product_id", + "Table": "order_items" + }, + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "count(*) * count(*) as count(*)", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "p_category_id": 1 + }, + "TableName": "products_categories", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), p.category_id from products as p where 1 != 1 group by p.category_id", + "Query": "select count(*), p.category_id from products as p where p.id = :oi_product_id group by p.category_id", + "Table": "products", + "Values": [ + ":oi_product_id" + ], + "Vindex": "xxhash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where 1 != 1 group by c.id, weight_string(c.id)", + "Query": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where c.id = :p_category_id group by c.id, weight_string(c.id)", + "Table": "categories", + "Values": [ + ":p_category_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } +``` -#### Q9 +#### Q9 (`Complex routed`) ```sql SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` +```json +{ + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "any_value(0) AS name, sum_count(1) AS order_count", + "GroupBy": "(2|3)", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "count(o.id) * count(*) as order_count", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "oi_product_id": 1 + }, + "TableName": "order_items_orders_products_categories", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(o.id) as order_count", + ":2 as product_id" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,L:1", + "JoinVars": { + "oi_order_id": 2 + }, + "TableName": "order_items_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), oi.product_id, oi.order_id from order_items as oi where 1 != 1 group by oi.product_id, oi.order_id", + "Query": "select count(*), oi.product_id, oi.order_id from order_items as oi group by oi.product_id, oi.order_id", + "Table": "order_items" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(o.id) as order_count from orders as o where 1 != 1 group by .0", + "Query": "select count(o.id) as order_count from orders as o where o.id = :oi_order_id group by .0", + "Table": "orders", + "Values": [ + ":oi_order_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + }, + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "count(*) * count(*) as count(*)", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "p_category_id": 1 + }, + "TableName": "products_categories", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), p.category_id from products as p where 1 != 1 group by p.category_id", + "Query": "select count(*), p.category_id from products as p where p.id = :oi_product_id group by p.category_id", + "Table": "products", + "Values": [ + ":oi_product_id" + ], + "Vindex": "xxhash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where 1 != 1 group by c.id, weight_string(c.id)", + "Query": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where c.id = :p_category_id group by c.id, weight_string(c.id)", + "Table": "categories", + "Values": [ + ":p_category_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } +``` -#### Q10 +#### Q10 (`Complex routed`) ```sql SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` +```json +{ + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum_count_star(1) AS order_count", + "GroupBy": "(0|2)", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select DATE(o.created_at) as order_date, count(*) as order_count, weight_string(DATE(o.created_at)) from orders as o where 1 != 1 group by DATE(o.created_at), weight_string(DATE(o.created_at))", + "OrderBy": "(0|2) ASC", + "Query": "select DATE(o.created_at) as order_date, count(*) as order_count, weight_string(DATE(o.created_at)) from orders as o where o.created_at \u003e= date_sub(now(), interval :1 day) group by DATE(o.created_at), weight_string(DATE(o.created_at)) order by DATE(o.created_at) asc", + "Table": "orders" + } + ] + } +``` ## Tables |Table Name|Reads|Writes| From fae18cb09704765503a32d2f97510df5c599170c Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 12 Dec 2024 11:41:56 -0600 Subject: [PATCH 4/7] Clean up summarize CLI code to use a Config struct Signed-off-by: Florent Poinsard --- go/cmd/summarize.go | 16 ++++++++-------- go/summarize/summarize.go | 25 ++++++++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/go/cmd/summarize.go b/go/cmd/summarize.go index 23d1547..458925a 100644 --- a/go/cmd/summarize.go +++ b/go/cmd/summarize.go @@ -22,10 +22,8 @@ import ( "github.com/vitessio/vt/go/summarize" ) -func summarizeCmd(port *int64) *cobra.Command { - var hotMetric string - var showGraph bool - var outputFormat string +func summarizeCmd(_ *int64) *cobra.Command { + var cfg summarize.Config cmd := &cobra.Command{ Use: "summarize old_file.json [new_file.json]", @@ -34,13 +32,15 @@ func summarizeCmd(port *int64) *cobra.Command { Example: "vt summarize old.json new.json", Args: cobra.RangeArgs(1, 2), Run: func(_ *cobra.Command, args []string) { - summarize.Run(args, hotMetric, showGraph, outputFormat, port) + cfg.Files = args + summarize.Run(&cfg) }, } - cmd.Flags().StringVar(&hotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)") - cmd.Flags().BoolVar(&showGraph, "graph", false, "Show the query graph in the browser") - cmd.Flags().StringVar(&outputFormat, "format", "html", "Output format (options: html, markdown)") + cmd.Flags().StringVar(&cfg.HotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)") + cmd.Flags().BoolVar(&cfg.ShowGraph, "graph", false, "Show the query graph in the browser") + cmd.Flags().StringVar(&cfg.OutputFormat, "format", "html", "Output format (options: html, markdown)") + cmd.Flags().IntVar(&cfg.Port, "port", 8000, "Port to listen on for the webserver") return cmd } diff --git a/go/summarize/summarize.go b/go/summarize/summarize.go index 51e6c22..8409f10 100644 --- a/go/summarize/summarize.go +++ b/go/summarize/summarize.go @@ -35,6 +35,17 @@ import ( ) type ( + Config struct { + Files []string + HotMetric string + + OutputFormat string + + Port int + + ShowGraph bool + } + traceSummary struct { Name string TracedQueries []TracedQuery @@ -43,11 +54,11 @@ type ( type summaryWorker = func(s *Summary) error -func Run(files []string, hotMetric string, showGraph bool, outputFormat string, port *int64) { +func Run(cfg *Config) { var traces []traceSummary var workers []summaryWorker - for _, file := range files { + for _, file := range cfg.Files { typ, err := data.GetFileType(file) exitIfError(err) var w summarizer @@ -78,16 +89,16 @@ func Run(files []string, hotMetric string, showGraph bool, outputFormat string, traceCount := len(traces) if traceCount <= 0 { - s, err := printSummary(hotMetric, workers, outputFormat, port) + s, err := printSummary(cfg.HotMetric, workers, cfg.OutputFormat, cfg.Port) exitIfError(err) - if showGraph { + if cfg.ShowGraph { err := renderQueryGraph(s) exitIfError(err) } return } - err := checkTraceConditions(traces, workers, hotMetric) + err := checkTraceConditions(traces, workers, cfg.HotMetric) exitIfError(err) switch traceCount { @@ -107,7 +118,7 @@ func exitIfError(err error) { os.Exit(1) } -func printSummary(hotMetric string, workers []summaryWorker, outputFormat string, port *int64) (*Summary, error) { +func printSummary(hotMetric string, workers []summaryWorker, outputFormat string, port int) (*Summary, error) { s, err := NewSummary(hotMetric) if err != nil { return nil, err @@ -124,7 +135,7 @@ func printSummary(hotMetric string, workers []summaryWorker, outputFormat string return nil, err } outputFormat = strings.ToLower(outputFormat) - if *port == 0 && outputFormat == "html" { + if port == 0 && outputFormat == "html" { fmt.Println("port is required when output format is html") os.Exit(1) } From e70d1db22cc3a2526132b923447039d4757b8c41 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 12 Dec 2024 14:47:30 -0600 Subject: [PATCH 5/7] Implement a central state for the web server Signed-off-by: Florent Poinsard --- go/cmd/root.go | 52 ++++++++++++++-------------- go/cmd/summarize.go | 8 +++-- go/summarize/summarize.go | 7 ++-- go/web/state/state.go | 72 +++++++++++++++++++++++++++++++++++++++ go/web/web.go | 14 ++++---- 5 files changed, 116 insertions(+), 37 deletions(-) create mode 100644 go/web/state/state.go diff --git a/go/cmd/root.go b/go/cmd/root.go index e3a99dc..8467052 100644 --- a/go/cmd/root.go +++ b/go/cmd/root.go @@ -24,32 +24,26 @@ import ( "github.com/spf13/cobra" "github.com/vitessio/vt/go/web" + "github.com/vitessio/vt/go/web/state" ) +//nolint:gochecknoglobals // the state is protected using mutexes +var wstate *state.State + // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { // rootCmd represents the base command when called without any subcommands - var port int64 - webserverStarted := false - ch := make(chan int, 2) root := &cobra.Command{ Use: "vt", Short: "Utils tools for testing, running and benchmarking Vitess.", - RunE: func(_ *cobra.Command, _ []string) error { - if port > 0 { - if webserverStarted { - return nil - } - webserverStarted = true - go startWebServer(port, ch) - <-ch - } - return nil + CompletionOptions: cobra.CompletionOptions{ + DisableDefaultCmd: true, }, } + + var port int64 root.PersistentFlags().Int64VarP(&port, "port", "p", 8080, "Port to run the web server on") - root.CompletionOptions.HiddenDefaultCmd = true root.AddCommand(summarizeCmd(&port)) root.AddCommand(testerCmd()) @@ -63,16 +57,20 @@ func Execute() { panic(err) } - if !webserverStarted && port > 0 { - webserverStarted = true - go startWebServer(port, ch) + // Start the web server for all commands no matter what + wstate = state.NewState(port) + ch := make(chan int, 1) + if port > 0 { + wstate.SetStarted(true) + go startWebServer(ch) + if !wstate.WaitUntilAvailable(10 * time.Second) { + fmt.Println("Timed out waiting for server to start") + os.Exit(1) + } } else { ch <- 1 } - // FIXME: add sync b/w webserver and root command, for now just add a wait to make sure webserver is running - time.Sleep(2 * time.Second) - err := root.Execute() if err != nil { fmt.Printf("Error: %v\n", err) @@ -81,13 +79,15 @@ func Execute() { <-ch } -func startWebServer(port int64, ch chan int) { - if port > 0 && port != 8080 { - panic("(FIXME: make port configurable) Port is not 8080") +func startWebServer(ch chan int) { + err := web.Run(wstate) + if err != nil { + panic(err) } - web.Run(port) - if os.WriteFile("/dev/stderr", []byte("Web server is running, use Ctrl-C to exit\n"), 0o600) != nil { - panic("Failed to write to /dev/stderr") + + _, err = fmt.Fprint(os.Stderr, "Web server is running, use Ctrl-C to exit\n") + if err != nil { + panic(err) } ch <- 1 } diff --git a/go/cmd/summarize.go b/go/cmd/summarize.go index 458925a..989ebb4 100644 --- a/go/cmd/summarize.go +++ b/go/cmd/summarize.go @@ -22,8 +22,10 @@ import ( "github.com/vitessio/vt/go/summarize" ) -func summarizeCmd(_ *int64) *cobra.Command { - var cfg summarize.Config +func summarizeCmd(port *int64) *cobra.Command { + cfg := summarize.Config{ + WState: wstate, + } cmd := &cobra.Command{ Use: "summarize old_file.json [new_file.json]", @@ -33,6 +35,7 @@ func summarizeCmd(_ *int64) *cobra.Command { Args: cobra.RangeArgs(1, 2), Run: func(_ *cobra.Command, args []string) { cfg.Files = args + cfg.Port = *port summarize.Run(&cfg) }, } @@ -40,7 +43,6 @@ func summarizeCmd(_ *int64) *cobra.Command { cmd.Flags().StringVar(&cfg.HotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)") cmd.Flags().BoolVar(&cfg.ShowGraph, "graph", false, "Show the query graph in the browser") cmd.Flags().StringVar(&cfg.OutputFormat, "format", "html", "Output format (options: html, markdown)") - cmd.Flags().IntVar(&cfg.Port, "port", 8000, "Port to listen on for the webserver") return cmd } diff --git a/go/summarize/summarize.go b/go/summarize/summarize.go index 8409f10..1f7cdcd 100644 --- a/go/summarize/summarize.go +++ b/go/summarize/summarize.go @@ -32,6 +32,7 @@ import ( "golang.org/x/term" "github.com/vitessio/vt/go/data" + "github.com/vitessio/vt/go/web/state" ) type ( @@ -41,9 +42,11 @@ type ( OutputFormat string - Port int + Port int64 ShowGraph bool + + WState *state.State } traceSummary struct { @@ -118,7 +121,7 @@ func exitIfError(err error) { os.Exit(1) } -func printSummary(hotMetric string, workers []summaryWorker, outputFormat string, port int) (*Summary, error) { +func printSummary(hotMetric string, workers []summaryWorker, outputFormat string, port int64) (*Summary, error) { s, err := NewSummary(hotMetric) if err != nil { return nil, err diff --git a/go/web/state/state.go b/go/web/state/state.go new file mode 100644 index 0000000..b0a1647 --- /dev/null +++ b/go/web/state/state.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "fmt" + "net/http" + "sync" + "time" +) + +type State struct { + mu sync.Mutex + + port int64 + started bool +} + +func NewState(port int64) *State { + return &State{ + port: port, + } +} + +func (s *State) SetStarted(v bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.started = v +} + +func (s *State) Started() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.started +} + +func (s *State) GetPort() int64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.port +} + +func (s *State) WaitUntilAvailable(timeout time.Duration) bool { + for { + select { + case <-time.After(timeout): + return false + case <-time.After(100 * time.Millisecond): + r, err := http.Get(fmt.Sprintf("http://localhost:%d/", s.port)) + if err != nil { + continue + } + if r.StatusCode == http.StatusOK { + return true + } + } + } +} diff --git a/go/web/web.go b/go/web/web.go index d765ca1..d3e476d 100644 --- a/go/web/web.go +++ b/go/web/web.go @@ -6,7 +6,6 @@ import ( "fmt" "html/template" "io" - "log" "net/http" "os" "time" @@ -14,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/vitessio/vt/go/summarize" + "github.com/vitessio/vt/go/web/state" ) func RenderFileToGin(fileName string, data any, c *gin.Context) { @@ -84,7 +84,7 @@ func addFuncMap(r *gin.Engine) { r.SetFuncMap(getFuncMap()) } -func Run(port int64) { +func Run(s *state.State) error { gin.SetMode(gin.ReleaseMode) gin.DefaultWriter = io.Discard // Disable logging r := gin.Default() @@ -125,10 +125,12 @@ func Run(port int64) { RenderFileToGin("summarize.html", &summarizeOutput, c) }) - if os.WriteFile("/dev/stderr", []byte(fmt.Sprintf("Starting web server on http://localhost:%d\n", port)), 0o600) != nil { - panic("Failed to write to /dev/stderr") + if _, err := fmt.Fprintf(os.Stderr, "Starting web server on http://localhost:%d\n", s.GetPort()); err != nil { + return err } - if err := r.Run(fmt.Sprintf(":%d", port)); err != nil { - log.Fatalf("Failed to start server: %v", err) + + if err := r.Run(fmt.Sprintf(":%d", s.GetPort())); err != nil { + return err } + return nil } From ff2bae4f02a0e7ad3e7e3eb15d6e719cb99b02a3 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 12 Dec 2024 15:03:44 -0600 Subject: [PATCH 6/7] Enhance HTML with new hotqueries and query plan general info Signed-off-by: Florent Poinsard --- go/summarize/markdown.go | 5 ++-- go/summarize/summarize-planalyze.go | 7 +++-- go/summarize/summarize.go | 14 ++++----- go/summarize/summary.go | 10 +++---- go/web/templates/summarize.html | 44 +++++++++++++++++++++++++---- 5 files changed, 57 insertions(+), 23 deletions(-) diff --git a/go/summarize/markdown.go b/go/summarize/markdown.go index 6b44f5d..ca8e557 100644 --- a/go/summarize/markdown.go +++ b/go/summarize/markdown.go @@ -228,8 +228,7 @@ func renderTransactions(md *markdown.MarkDown, transactions []TransactionSummary } func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error { - sum := analysis.PassThrough + analysis.SimpleRouted + analysis.Complex + analysis.Unplannable - if sum == 0 { + if analysis.Total == 0 { return nil } @@ -241,7 +240,7 @@ func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error { {planalyze.SimpleRouted.String(), strconv.Itoa(analysis.SimpleRouted)}, {planalyze.Complex.String(), strconv.Itoa(analysis.Complex)}, {planalyze.Unplannable.String(), strconv.Itoa(analysis.Unplannable)}, - {"Total", strconv.Itoa(sum)}, + {"Total", strconv.Itoa(analysis.Total)}, } md.PrintTable(headers, rows) md.NewLine() diff --git a/go/summarize/summarize-planalyze.go b/go/summarize/summarize-planalyze.go index 0e8f26f..ebf79a2 100644 --- a/go/summarize/summarize-planalyze.go +++ b/go/summarize/summarize-planalyze.go @@ -21,17 +21,18 @@ import ( ) func summarizePlanAnalyze(s *Summary, data planalyze.Output) (err error) { - s.planAnalysis = PlanAnalysis{ + s.PlanAnalysis = PlanAnalysis{ PassThrough: len(data.PassThrough), SimpleRouted: len(data.SimpleRouted), Complex: len(data.Complex), Unplannable: len(data.Unplannable), } + s.PlanAnalysis.Total = s.PlanAnalysis.PassThrough + s.PlanAnalysis.SimpleRouted + s.PlanAnalysis.Complex + s.PlanAnalysis.Unplannable s.addPlanResult(data.SimpleRouted) s.addPlanResult(data.Complex) - s.planAnalysis.simpleRouted = append(s.planAnalysis.simpleRouted, data.SimpleRouted...) - s.planAnalysis.complex = append(s.planAnalysis.complex, data.Complex...) + s.PlanAnalysis.simpleRouted = append(s.PlanAnalysis.simpleRouted, data.SimpleRouted...) + s.PlanAnalysis.complex = append(s.PlanAnalysis.complex, data.Complex...) return nil } diff --git a/go/summarize/summarize.go b/go/summarize/summarize.go index 1f7cdcd..1ef90a8 100644 --- a/go/summarize/summarize.go +++ b/go/summarize/summarize.go @@ -179,26 +179,26 @@ func compileSummary(s *Summary) error { func compileHotQueries(s *Summary) error { for _, result := range s.queries { - checkQueryForHotness(&s.hotQueries, result, s.hotQueryFn) + checkQueryForHotness(&s.HotQueries, result, s.hotQueryFn) } var hasTime bool - sort.Slice(s.hotQueries, func(i, j int) bool { - if s.hotQueries[i].QueryAnalysisResult.QueryTime != 0 { + sort.Slice(s.HotQueries, func(i, j int) bool { + if s.HotQueries[i].QueryAnalysisResult.QueryTime != 0 { hasTime = true } - fnI := s.hotQueryFn(s.hotQueries[i].QueryAnalysisResult) - fnJ := s.hotQueryFn(s.hotQueries[j].QueryAnalysisResult) + fnI := s.hotQueryFn(s.HotQueries[i].QueryAnalysisResult) + fnJ := s.hotQueryFn(s.HotQueries[j].QueryAnalysisResult) // if the two metrics are equal, sort them by alphabetical order if fnI == fnJ { - return s.hotQueries[i].QueryAnalysisResult.QueryStructure > s.hotQueries[j].QueryAnalysisResult.QueryStructure + return s.HotQueries[i].QueryAnalysisResult.QueryStructure > s.HotQueries[j].QueryAnalysisResult.QueryStructure } return fnI > fnJ }) // If we did not record any time, there is no hotness to record, so removing the field so it does not get rendered. if !hasTime { - s.hotQueries = nil + s.HotQueries = nil } return nil } diff --git a/go/summarize/summary.go b/go/summarize/summary.go index e718a0f..a4cd613 100644 --- a/go/summarize/summary.go +++ b/go/summarize/summary.go @@ -35,9 +35,8 @@ type ( Tables []*TableSummary Failures []FailuresSummary Transactions []TransactionSummary - HotQueries []keys.QueryAnalysisResult - planAnalysis PlanAnalysis - hotQueries []HotQueryResult + PlanAnalysis PlanAnalysis + HotQueries []HotQueryResult hotQueryFn getMetric AnalyzedFiles []string queryGraph queryGraph @@ -88,6 +87,7 @@ type ( SimpleRouted int Complex int Unplannable int + Total int simpleRouted []planalyze.AnalyzedQuery complex []planalyze.AnalyzedQuery @@ -137,11 +137,11 @@ func (s *Summary) PrintMarkdown(out io.Writer, now time.Time) error { s.AnalyzedFiles[i] = "`" + file + "`" } md.Printf(msg, now.Format(time.DateTime), filePlural, strings.Join(s.AnalyzedFiles, ", ")) - err := renderPlansSection(md, s.planAnalysis) + err := renderPlansSection(md, s.PlanAnalysis) if err != nil { return err } - renderHotQueries(md, s.hotQueries) + renderHotQueries(md, s.HotQueries) renderTableUsage(md, s.Tables, s.HasRowCount) renderTablesJoined(md, s) renderTransactions(md, s.Transactions) diff --git a/go/web/templates/summarize.html b/go/web/templates/summarize.html index b31d7e0..8d86e97 100644 --- a/go/web/templates/summarize.html +++ b/go/web/templates/summarize.html @@ -13,6 +13,40 @@

Query Analysis Report

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Plan ComplexityCount
Pass-through{{.PlanAnalysis.PassThrough}}
Simple routed{{.PlanAnalysis.SimpleRouted}}
Complex routed{{.PlanAnalysis.Complex}}
Unplannable{{.PlanAnalysis.Unplannable}}
Total{{.PlanAnalysis.Total}}
+
+
@@ -29,10 +63,10 @@

Query Analysis Report

{{range $index, $query := .HotQueries}} - - - - + + + + {{end}} @@ -52,7 +86,7 @@

Query Analysis Report

{{range $index, $query := .HotQueries}} - + {{end}} From aa1ba18e986a3582fa23339c5987984a395e3674 Mon Sep 17 00:00:00 2001 From: Florent Poinsard Date: Thu, 12 Dec 2024 15:23:36 -0600 Subject: [PATCH 7/7] Add query plans to the html Signed-off-by: Florent Poinsard --- go/summarize/markdown.go | 4 +-- go/summarize/summarize-planalyze.go | 4 +-- go/summarize/summary.go | 4 +-- go/web/templates/summarize.html | 41 +++++++++++++++++++++++++++++ go/web/web.go | 5 ++++ 5 files changed, 52 insertions(+), 6 deletions(-) diff --git a/go/summarize/markdown.go b/go/summarize/markdown.go index ca8e557..b656356 100644 --- a/go/summarize/markdown.go +++ b/go/summarize/markdown.go @@ -245,11 +245,11 @@ func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error { md.PrintTable(headers, rows) md.NewLine() - err := renderQueryPlans(md, analysis.simpleRouted, planalyze.SimpleRouted.String()) + err := renderQueryPlans(md, analysis.SimpleRoutedQ, planalyze.SimpleRouted.String()) if err != nil { return err } - return renderQueryPlans(md, analysis.complex, planalyze.Complex.String()) + return renderQueryPlans(md, analysis.ComplexQ, planalyze.Complex.String()) } func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery, title string) error { diff --git a/go/summarize/summarize-planalyze.go b/go/summarize/summarize-planalyze.go index ebf79a2..88f1a42 100644 --- a/go/summarize/summarize-planalyze.go +++ b/go/summarize/summarize-planalyze.go @@ -32,7 +32,7 @@ func summarizePlanAnalyze(s *Summary, data planalyze.Output) (err error) { s.addPlanResult(data.SimpleRouted) s.addPlanResult(data.Complex) - s.PlanAnalysis.simpleRouted = append(s.PlanAnalysis.simpleRouted, data.SimpleRouted...) - s.PlanAnalysis.complex = append(s.PlanAnalysis.complex, data.Complex...) + s.PlanAnalysis.SimpleRoutedQ = append(s.PlanAnalysis.SimpleRoutedQ, data.SimpleRouted...) + s.PlanAnalysis.ComplexQ = append(s.PlanAnalysis.ComplexQ, data.Complex...) return nil } diff --git a/go/summarize/summary.go b/go/summarize/summary.go index a4cd613..0f0cecc 100644 --- a/go/summarize/summary.go +++ b/go/summarize/summary.go @@ -89,8 +89,8 @@ type ( Unplannable int Total int - simpleRouted []planalyze.AnalyzedQuery - complex []planalyze.AnalyzedQuery + SimpleRoutedQ []planalyze.AnalyzedQuery + ComplexQ []planalyze.AnalyzedQuery } ) diff --git a/go/web/templates/summarize.html b/go/web/templates/summarize.html index 8d86e97..00a44bd 100644 --- a/go/web/templates/summarize.html +++ b/go/web/templates/summarize.html @@ -47,6 +47,47 @@

Query Analysis Report

{{$index | add 1}} {{$query.UsageCount}}{{$query.QueryTime}}{{divide .QueryTime .UsageCount}}{{$query.RowsExamined}}{{$query.QueryAnalysisResult.UsageCount}}{{$query.QueryAnalysisResult.QueryTime}}{{divide $query.QueryAnalysisResult.QueryTime $query.QueryAnalysisResult.UsageCount}}{{$query.QueryAnalysisResult.RowsExamined}}
{{$index | add 1}}{{.QueryStructure}}{{$query.QueryAnalysisResult.QueryStructure}}
+ +
+

Simple Routed Queries

+ {{range $index, $query := .PlanAnalysis.SimpleRoutedQ}} + + + + + + + + + + + +
Query
{{$query.QueryStructure}}
+ 
+        {{- jsonToString $query.PlanOutput}}
+
+ {{end}} + +

Complex Queries

+ {{range $index, $query := .PlanAnalysis.ComplexQ}} + + + + + + + + + + + +
Query
{{$query.QueryStructure}}
+ 
+        {{- jsonToString $query.PlanOutput}}
+
+ {{end}} +
+
diff --git a/go/web/web.go b/go/web/web.go index d3e476d..c52af58 100644 --- a/go/web/web.go +++ b/go/web/web.go @@ -48,6 +48,11 @@ type SummaryOutput struct { func getFuncMap() template.FuncMap { return template.FuncMap{ + "jsonToString": func(j json.RawMessage) string { + var formattedJSON bytes.Buffer + _ = json.Indent(&formattedJSON, j, "", " ") + return formattedJSON.String() + }, "add": func(a, b int) int { return a + b }, "divide": func(a, b any) float64 { if b == 0 || b == nil {