diff --git a/go/cmd/root.go b/go/cmd/root.go index e3a99dc..8467052 100644 --- a/go/cmd/root.go +++ b/go/cmd/root.go @@ -24,32 +24,26 @@ import ( "github.com/spf13/cobra" "github.com/vitessio/vt/go/web" + "github.com/vitessio/vt/go/web/state" ) +//nolint:gochecknoglobals // the state is protected using mutexes +var wstate *state.State + // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { // rootCmd represents the base command when called without any subcommands - var port int64 - webserverStarted := false - ch := make(chan int, 2) root := &cobra.Command{ Use: "vt", Short: "Utils tools for testing, running and benchmarking Vitess.", - RunE: func(_ *cobra.Command, _ []string) error { - if port > 0 { - if webserverStarted { - return nil - } - webserverStarted = true - go startWebServer(port, ch) - <-ch - } - return nil + CompletionOptions: cobra.CompletionOptions{ + DisableDefaultCmd: true, }, } + + var port int64 root.PersistentFlags().Int64VarP(&port, "port", "p", 8080, "Port to run the web server on") - root.CompletionOptions.HiddenDefaultCmd = true root.AddCommand(summarizeCmd(&port)) root.AddCommand(testerCmd()) @@ -63,16 +57,20 @@ func Execute() { panic(err) } - if !webserverStarted && port > 0 { - webserverStarted = true - go startWebServer(port, ch) + // Start the web server for all commands no matter what + wstate = state.NewState(port) + ch := make(chan int, 1) + if port > 0 { + wstate.SetStarted(true) + go startWebServer(ch) + if !wstate.WaitUntilAvailable(10 * time.Second) { + fmt.Println("Timed out waiting for server to start") + os.Exit(1) + } } else { ch <- 1 } - // FIXME: add sync b/w webserver and root command, for now just add a wait to make sure webserver is running - time.Sleep(2 * time.Second) - err := root.Execute() if err != nil { fmt.Printf("Error: %v\n", err) @@ -81,13 +79,15 @@ func Execute() { <-ch } -func startWebServer(port int64, ch chan int) { - if port > 0 && port != 8080 { - panic("(FIXME: make port configurable) Port is not 8080") +func startWebServer(ch chan int) { + err := web.Run(wstate) + if err != nil { + panic(err) } - web.Run(port) - if os.WriteFile("/dev/stderr", []byte("Web server is running, use Ctrl-C to exit\n"), 0o600) != nil { - panic("Failed to write to /dev/stderr") + + _, err = fmt.Fprint(os.Stderr, "Web server is running, use Ctrl-C to exit\n") + if err != nil { + panic(err) } ch <- 1 } diff --git a/go/cmd/summarize.go b/go/cmd/summarize.go index 23d1547..989ebb4 100644 --- a/go/cmd/summarize.go +++ b/go/cmd/summarize.go @@ -23,9 +23,9 @@ import ( ) func summarizeCmd(port *int64) *cobra.Command { - var hotMetric string - var showGraph bool - var outputFormat string + cfg := summarize.Config{ + WState: wstate, + } cmd := &cobra.Command{ Use: "summarize old_file.json [new_file.json]", @@ -34,13 +34,15 @@ func summarizeCmd(port *int64) *cobra.Command { Example: "vt summarize old.json new.json", Args: cobra.RangeArgs(1, 2), Run: func(_ *cobra.Command, args []string) { - summarize.Run(args, hotMetric, showGraph, outputFormat, port) + cfg.Files = args + cfg.Port = *port + summarize.Run(&cfg) }, } - cmd.Flags().StringVar(&hotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)") - cmd.Flags().BoolVar(&showGraph, "graph", false, "Show the query graph in the browser") - cmd.Flags().StringVar(&outputFormat, "format", "html", "Output format (options: html, markdown)") + cmd.Flags().StringVar(&cfg.HotMetric, "hot-metric", "total-time", "Metric to determine hot queries (options: usage-count, total-rows-examined, avg-rows-examined, avg-time, total-time)") + cmd.Flags().BoolVar(&cfg.ShowGraph, "graph", false, "Show the query graph in the browser") + cmd.Flags().StringVar(&cfg.OutputFormat, "format", "html", "Output format (options: html, markdown)") return cmd } diff --git a/go/summarize/markdown.go b/go/summarize/markdown.go index 96afdab..b656356 100644 --- a/go/summarize/markdown.go +++ b/go/summarize/markdown.go @@ -27,29 +27,15 @@ import ( humanize "github.com/dustin/go-humanize" - "github.com/vitessio/vt/go/keys" "github.com/vitessio/vt/go/markdown" "github.com/vitessio/vt/go/planalyze" ) -func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, metricReader getMetric) { +func renderHotQueries(md *markdown.MarkDown, queries []HotQueryResult) { if len(queries) == 0 { return } - hasTime := false - // Sort the queries in descending order of hotness - sort.Slice(queries, func(i, j int) bool { - if queries[i].QueryTime != 0 { - hasTime = true - } - return metricReader(queries[i]) > metricReader(queries[j]) - }) - - if !hasTime { - return - } - md.PrintHeader("Top Queries", 2) // Prepare table headers and rows @@ -58,13 +44,12 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, for i, query := range queries { queryID := fmt.Sprintf("Q%d", i+1) - avgQueryTime := query.QueryTime / float64(query.UsageCount) rows = append(rows, []string{ queryID, - humanize.Comma(int64(query.UsageCount)), - fmt.Sprintf("%.2f", query.QueryTime), - fmt.Sprintf("%.2f", avgQueryTime), - humanize.Comma(int64(query.RowsExamined)), + humanize.Comma(int64(query.QueryAnalysisResult.UsageCount)), + fmt.Sprintf("%.2f", query.QueryAnalysisResult.QueryTime), + fmt.Sprintf("%.2f", query.AvgQueryTime), + humanize.Comma(int64(query.QueryAnalysisResult.RowsExamined)), }) } @@ -74,11 +59,24 @@ func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, // After the table, list the full queries with their IDs md.PrintHeader("Query Details", 3) for i, query := range queries { + hasPlanAnalysis := len(string(query.PlanAnalysis.PlanOutput)) > 0 + queryID := fmt.Sprintf("Q%d", i+1) + if hasPlanAnalysis { + queryID += fmt.Sprintf(" (`%s`)", query.PlanAnalysis.Complexity.String()) + } + md.PrintHeader(queryID, 4) md.Println("```sql") - md.Println(query.QueryStructure) + md.Println(query.QueryAnalysisResult.QueryStructure) md.Println("```") + + if hasPlanAnalysis { + md.Println("```json") + md.Println(string(query.PlanAnalysis.PlanOutput)) + md.Println("```") + } + md.NewLine() } } @@ -230,8 +228,7 @@ func renderTransactions(md *markdown.MarkDown, transactions []TransactionSummary } func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error { - sum := analysis.PassThrough + analysis.SimpleRouted + analysis.Complex + analysis.Unplannable - if sum == 0 { + if analysis.Total == 0 { return nil } @@ -243,25 +240,28 @@ func renderPlansSection(md *markdown.MarkDown, analysis PlanAnalysis) error { {planalyze.SimpleRouted.String(), strconv.Itoa(analysis.SimpleRouted)}, {planalyze.Complex.String(), strconv.Itoa(analysis.Complex)}, {planalyze.Unplannable.String(), strconv.Itoa(analysis.Unplannable)}, - {"Total", strconv.Itoa(sum)}, + {"Total", strconv.Itoa(analysis.Total)}, } md.PrintTable(headers, rows) md.NewLine() - err := renderQueryPlans(md, analysis.simpleRouted, planalyze.SimpleRouted.String()) + err := renderQueryPlans(md, analysis.SimpleRoutedQ, planalyze.SimpleRouted.String()) if err != nil { return err } - return renderQueryPlans(md, analysis.complex, planalyze.Complex.String()) + return renderQueryPlans(md, analysis.ComplexQ, planalyze.Complex.String()) } func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery, title string) error { for i, query := range queries { if i == 0 { - md.Printf("# %s Queries\n\n", title) + md.PrintHeader(fmt.Sprintf("%s Queries\n\n", title), 3) } - md.Printf("## Query\n\n```sql\n%s\n```\n\n", query.QueryStructure) - md.Println("## Plan\n\n```json") + md.PrintHeader("Query", 4) + md.Printf("```sql\n%s\n```\n\n", query.QueryStructure) + + md.PrintHeader("Plan", 4) + md.Println("```json") // Indent the JSON output. If we don't do this, the json will be indented all wrong var buf bytes.Buffer @@ -274,6 +274,7 @@ func renderQueryPlans(md *markdown.MarkDown, queries []planalyze.AnalyzedQuery, } md.NewLine() md.Println("```") + md.Println("---") md.NewLine() } return nil diff --git a/go/summarize/summarize-keys.go b/go/summarize/summarize-keys.go index a897377..d9a69da 100644 --- a/go/summarize/summarize-keys.go +++ b/go/summarize/summarize-keys.go @@ -247,8 +247,8 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error { // First pass: collect all graphData and count occurrences for _, query := range queries.Queries { + summary.addQueryResult(query) gatherTableInfo(query, tableSummaries, tableUsageWriteCounts, tableUsageReadCounts) - checkQueryForHotness(&summary.HotQueries, query, summary.hotQueryFn) } // Second pass: calculate percentages @@ -327,15 +327,25 @@ func summarizeKeysQueries(summary *Summary, queries *keys.Output) error { return nil } -func checkQueryForHotness(hotQueries *[]keys.QueryAnalysisResult, query keys.QueryAnalysisResult, metricReader getMetric) { +func checkQueryForHotness(hotQueries *[]HotQueryResult, query QueryResult, metricReader getMetric) { // todo: we should be able to choose different metrics for hotness - e.g. total time spent on query, number of rows examined, etc. + newHotQueryFn := func() HotQueryResult { + return HotQueryResult{ + QueryResult: QueryResult{ + QueryAnalysisResult: query.QueryAnalysisResult, + PlanAnalysis: query.PlanAnalysis, + }, + AvgQueryTime: query.QueryAnalysisResult.QueryTime / float64(query.QueryAnalysisResult.UsageCount), + } + } + switch { case len(*hotQueries) < HotQueryCount: // If we have not yet reached the limit, add the query - *hotQueries = append(*hotQueries, query) - case metricReader(query) > metricReader((*hotQueries)[0]): + *hotQueries = append(*hotQueries, newHotQueryFn()) + case metricReader(query.QueryAnalysisResult) > metricReader((*hotQueries)[0].QueryAnalysisResult): // If the current query has more usage than the least used hot query, replace it - (*hotQueries)[0] = query + (*hotQueries)[0] = newHotQueryFn() default: // If the current query is not hot enough, just return return @@ -344,7 +354,7 @@ func checkQueryForHotness(hotQueries *[]keys.QueryAnalysisResult, query keys.Que // Sort the hot queries by query time so that the least used query is always at the front sort.Slice(*hotQueries, func(i, j int) bool { - return metricReader((*hotQueries)[i]) < metricReader((*hotQueries)[j]) + return metricReader((*hotQueries)[i].QueryAnalysisResult) < metricReader((*hotQueries)[j].QueryAnalysisResult) }) } diff --git a/go/summarize/summarize-keys_test.go b/go/summarize/summarize-keys_test.go index b0a649f..76dc9cf 100644 --- a/go/summarize/summarize-keys_test.go +++ b/go/summarize/summarize-keys_test.go @@ -118,6 +118,9 @@ func TestSummarizeKeysWithHotnessFile(t *testing.T) { err = fn(s) require.NoError(t, err) + err = compileSummary(s) + require.NoError(t, err) + err = s.PrintMarkdown(sb, now) require.NoError(t, err) diff --git a/go/summarize/summarize-planalyze.go b/go/summarize/summarize-planalyze.go index ddda77c..88f1a42 100644 --- a/go/summarize/summarize-planalyze.go +++ b/go/summarize/summarize-planalyze.go @@ -21,14 +21,18 @@ import ( ) func summarizePlanAnalyze(s *Summary, data planalyze.Output) (err error) { - s.planAnalysis = PlanAnalysis{ + s.PlanAnalysis = PlanAnalysis{ PassThrough: len(data.PassThrough), SimpleRouted: len(data.SimpleRouted), Complex: len(data.Complex), Unplannable: len(data.Unplannable), } + s.PlanAnalysis.Total = s.PlanAnalysis.PassThrough + s.PlanAnalysis.SimpleRouted + s.PlanAnalysis.Complex + s.PlanAnalysis.Unplannable - s.planAnalysis.simpleRouted = append(s.planAnalysis.simpleRouted, data.SimpleRouted...) - s.planAnalysis.complex = append(s.planAnalysis.complex, data.Complex...) + s.addPlanResult(data.SimpleRouted) + s.addPlanResult(data.Complex) + + s.PlanAnalysis.SimpleRoutedQ = append(s.PlanAnalysis.SimpleRoutedQ, data.SimpleRouted...) + s.PlanAnalysis.ComplexQ = append(s.PlanAnalysis.ComplexQ, data.Complex...) return nil } diff --git a/go/summarize/summarize-planalyze_test.go b/go/summarize/summarize-planalyze_test.go index ace0721..f6ca648 100644 --- a/go/summarize/summarize-planalyze_test.go +++ b/go/summarize/summarize-planalyze_test.go @@ -27,15 +27,25 @@ import ( ) func TestSummarizePlans(t *testing.T) { - fn, err := readPlanalyzeFile("../testdata/planalyze-output/bigger_slow_query_plan_report.json") + fnPlan, err := readPlanalyzeFile("../testdata/planalyze-output/bigger_slow_query_plan_report.json") require.NoError(t, err) + + fnKeys, err := readKeysFile("../testdata/keys-output/bigger_slow_query_log.json") + require.NoError(t, err) + sb := &strings.Builder{} now := time.Date(2024, time.January, 1, 1, 2, 3, 0, time.UTC) - s, err := NewSummary("") + s, err := NewSummary("usage-count") + require.NoError(t, err) + + err = fnPlan(s) + require.NoError(t, err) + + err = fnKeys(s) require.NoError(t, err) - err = fn(s) + err = compileSummary(s) require.NoError(t, err) err = s.PrintMarkdown(sb, now) diff --git a/go/summarize/summarize.go b/go/summarize/summarize.go index fd33b5c..1ef90a8 100644 --- a/go/summarize/summarize.go +++ b/go/summarize/summarize.go @@ -23,6 +23,7 @@ import ( "io" "os" "os/exec" + "sort" "strings" "time" @@ -31,9 +32,23 @@ import ( "golang.org/x/term" "github.com/vitessio/vt/go/data" + "github.com/vitessio/vt/go/web/state" ) type ( + Config struct { + Files []string + HotMetric string + + OutputFormat string + + Port int64 + + ShowGraph bool + + WState *state.State + } + traceSummary struct { Name string TracedQueries []TracedQuery @@ -42,11 +57,11 @@ type ( type summaryWorker = func(s *Summary) error -func Run(files []string, hotMetric string, showGraph bool, outputFormat string, port *int64) { +func Run(cfg *Config) { var traces []traceSummary var workers []summaryWorker - for _, file := range files { + for _, file := range cfg.Files { typ, err := data.GetFileType(file) exitIfError(err) var w summarizer @@ -77,16 +92,16 @@ func Run(files []string, hotMetric string, showGraph bool, outputFormat string, traceCount := len(traces) if traceCount <= 0 { - s, err := printSummary(hotMetric, workers, outputFormat, port) + s, err := printSummary(cfg.HotMetric, workers, cfg.OutputFormat, cfg.Port) exitIfError(err) - if showGraph { + if cfg.ShowGraph { err := renderQueryGraph(s) exitIfError(err) } return } - err := checkTraceConditions(traces, workers, hotMetric) + err := checkTraceConditions(traces, workers, cfg.HotMetric) exitIfError(err) switch traceCount { @@ -106,7 +121,7 @@ func exitIfError(err error) { os.Exit(1) } -func printSummary(hotMetric string, workers []summaryWorker, outputFormat string, port *int64) (*Summary, error) { +func printSummary(hotMetric string, workers []summaryWorker, outputFormat string, port int64) (*Summary, error) { s, err := NewSummary(hotMetric) if err != nil { return nil, err @@ -117,8 +132,13 @@ func printSummary(hotMetric string, workers []summaryWorker, outputFormat string return nil, err } } + + err = compileSummary(s) + if err != nil { + return nil, err + } outputFormat = strings.ToLower(outputFormat) - if *port == 0 && outputFormat == "html" { + if port == 0 && outputFormat == "html" { fmt.Println("port is required when output format is html") os.Exit(1) } @@ -150,6 +170,39 @@ func printSummary(hotMetric string, workers []summaryWorker, outputFormat string return s, nil } +func compileSummary(s *Summary) error { + if err := compileHotQueries(s); err != nil { + return err + } + return nil +} + +func compileHotQueries(s *Summary) error { + for _, result := range s.queries { + checkQueryForHotness(&s.HotQueries, result, s.hotQueryFn) + } + var hasTime bool + sort.Slice(s.HotQueries, func(i, j int) bool { + if s.HotQueries[i].QueryAnalysisResult.QueryTime != 0 { + hasTime = true + } + fnI := s.hotQueryFn(s.HotQueries[i].QueryAnalysisResult) + fnJ := s.hotQueryFn(s.HotQueries[j].QueryAnalysisResult) + + // if the two metrics are equal, sort them by alphabetical order + if fnI == fnJ { + return s.HotQueries[i].QueryAnalysisResult.QueryStructure > s.HotQueries[j].QueryAnalysisResult.QueryStructure + } + return fnI > fnJ + }) + + // If we did not record any time, there is no hotness to record, so removing the field so it does not get rendered. + if !hasTime { + s.HotQueries = nil + } + return nil +} + func launchInBrowser(tmpFile *os.File) error { port := int64(8080) // FIXME: take this from flags url := fmt.Sprintf("http://localhost:%d/summarize?file=", port) + tmpFile.Name() diff --git a/go/summarize/summary.go b/go/summarize/summary.go index 2832912..0f0cecc 100644 --- a/go/summarize/summary.go +++ b/go/summarize/summary.go @@ -35,13 +35,24 @@ type ( Tables []*TableSummary Failures []FailuresSummary Transactions []TransactionSummary - HotQueries []keys.QueryAnalysisResult - planAnalysis PlanAnalysis + PlanAnalysis PlanAnalysis + HotQueries []HotQueryResult hotQueryFn getMetric AnalyzedFiles []string queryGraph queryGraph Joins []joinDetails HasRowCount bool + queries map[string]QueryResult + } + + QueryResult struct { + QueryAnalysisResult keys.QueryAnalysisResult + PlanAnalysis planalyze.AnalyzedQuery + } + + HotQueryResult struct { + QueryResult + AvgQueryTime float64 } TableSummary struct { @@ -76,9 +87,10 @@ type ( SimpleRouted int Complex int Unplannable int + Total int - simpleRouted []planalyze.AnalyzedQuery - complex []planalyze.AnalyzedQuery + SimpleRoutedQ []planalyze.AnalyzedQuery + ComplexQ []planalyze.AnalyzedQuery } ) @@ -90,10 +102,25 @@ func NewSummary(hotMetric string) (*Summary, error) { return &Summary{ queryGraph: make(queryGraph), + queries: make(map[string]QueryResult), hotQueryFn: hotness, }, nil } +func (s *Summary) addQueryResult(qr keys.QueryAnalysisResult) { + val := s.queries[qr.QueryStructure] + val.QueryAnalysisResult = qr + s.queries[qr.QueryStructure] = val +} + +func (s *Summary) addPlanResult(p []planalyze.AnalyzedQuery) { + for _, query := range p { + val := s.queries[query.QueryStructure] + val.PlanAnalysis = query + s.queries[query.QueryStructure] = val + } +} + func (s *Summary) PrintMarkdown(out io.Writer, now time.Time) error { md := &markdown.MarkDown{} filePlural := "" @@ -110,11 +137,11 @@ func (s *Summary) PrintMarkdown(out io.Writer, now time.Time) error { s.AnalyzedFiles[i] = "`" + file + "`" } md.Printf(msg, now.Format(time.DateTime), filePlural, strings.Join(s.AnalyzedFiles, ", ")) - err := renderPlansSection(md, s.planAnalysis) + err := renderPlansSection(md, s.PlanAnalysis) if err != nil { return err } - renderHotQueries(md, s.HotQueries, s.hotQueryFn) + renderHotQueries(md, s.HotQueries) renderTableUsage(md, s.Tables, s.HasRowCount) renderTablesJoined(md, s) renderTransactions(md, s.Transactions) diff --git a/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md b/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md index 1b717e5..0032ec0 100644 --- a/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md +++ b/go/testdata/summarize-output/bigger_slow_log_avg-rows-examined.md @@ -6,12 +6,12 @@ ## Top Queries |Query ID|Usage Count|Total Query Time (ms)|Avg Query Time (ms)|Total Rows Examined| |---|---|---|---|---| -|Q1|2|0.40|0.20|20,000| -|Q2|3|0.61|0.20|30,000| -|Q3|1|0.22|0.22|8,000| -|Q4|2|0.37|0.19|16,000| +|Q1|3|0.61|0.20|30,000| +|Q2|2|0.40|0.20|20,000| +|Q3|2|0.49|0.25|16,000| +|Q4|1|0.22|0.22|8,000| |Q5|2|0.37|0.19|16,000| -|Q6|2|0.49|0.25|16,000| +|Q6|2|0.37|0.19|16,000| |Q7|2|0.31|0.16|15,000| |Q8|1|0.20|0.20|6,500| |Q9|3|0.58|0.19|17,000| @@ -20,32 +20,32 @@ ### Query Details #### Q1 ```sql -SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ +SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ ``` #### Q2 ```sql -SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ +SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` #### Q3 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` JOIN `reviews` AS `r` ON `u`.`id` = `r`.`user_id` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) AND `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` #### Q4 ```sql -SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` JOIN `reviews` AS `r` ON `u`.`id` = `r`.`user_id` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) AND `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) ``` #### Q5 ```sql -SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` #### Q6 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` #### Q7 diff --git a/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md b/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md index 83d48eb..64de635 100644 --- a/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md +++ b/go/testdata/summarize-output/bigger_slow_log_total-rows-examined.md @@ -9,9 +9,9 @@ |Q1|3|0.61|0.20|30,000| |Q2|2|0.40|0.20|20,000| |Q3|3|0.58|0.19|17,000| -|Q4|2|0.37|0.19|16,000| +|Q4|2|0.49|0.25|16,000| |Q5|2|0.37|0.19|16,000| -|Q6|2|0.49|0.25|16,000| +|Q6|2|0.37|0.19|16,000| |Q7|2|0.31|0.16|15,000| |Q8|2|0.34|0.17|8,500| |Q9|1|0.22|0.22|8,000| @@ -35,17 +35,17 @@ SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS #### Q4 ```sql -SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` #### Q5 ```sql -SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` #### Q6 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` #### Q7 diff --git a/go/testdata/summarize-output/bigger_slow_log_usage-count.md b/go/testdata/summarize-output/bigger_slow_log_usage-count.md index e195fa0..995006c 100644 --- a/go/testdata/summarize-output/bigger_slow_log_usage-count.md +++ b/go/testdata/summarize-output/bigger_slow_log_usage-count.md @@ -8,14 +8,14 @@ |---|---|---|---|---| |Q1|3|0.58|0.19|17,000| |Q2|3|0.61|0.20|30,000| -|Q3|2|0.21|0.11|3,000| -|Q4|2|0.37|0.19|16,000| -|Q5|2|0.31|0.16|15,000| -|Q6|2|0.40|0.20|20,000| +|Q3|2|0.49|0.25|16,000| +|Q4|2|0.33|0.17|6,000| +|Q5|2|0.21|0.11|3,000| +|Q6|2|0.31|0.16|15,000| |Q7|2|0.34|0.17|8,500| -|Q8|2|0.33|0.17|6,000| +|Q8|2|0.40|0.20|20,000| |Q9|2|0.37|0.19|16,000| -|Q10|2|0.49|0.25|16,000| +|Q10|2|0.37|0.19|16,000| ### Query Details #### Q1 @@ -30,22 +30,22 @@ SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` #### Q3 ```sql -SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` #### Q4 ```sql -SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` ``` #### Q5 ```sql -SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ +SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ ``` #### Q6 ```sql -SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ +SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ ``` #### Q7 @@ -55,17 +55,17 @@ SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS #### Q8 ```sql -SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` +SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` #### Q9 ```sql -SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` #### Q10 ```sql -SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` ## Tables diff --git a/go/testdata/summarize-output/bigger_slow_query_plan_report.md b/go/testdata/summarize-output/bigger_slow_query_plan_report.md index 9b03400..c7baff1 100644 --- a/go/testdata/summarize-output/bigger_slow_query_plan_report.md +++ b/go/testdata/summarize-output/bigger_slow_query_plan_report.md @@ -1,7 +1,7 @@ # Query Analysis Report **Date of Analysis**: 2024-01-01 01:02:03 -**Analyzed File**: `../testdata/planalyze-output/bigger_slow_query_plan_report.json` +**Analyzed Files**: `../testdata/planalyze-output/bigger_slow_query_plan_report.json`, `../testdata/keys-output/bigger_slow_query_log.json` ## Query Planning Report |Plan Complexity|Count| @@ -13,16 +13,15 @@ |Total|13| -# Simple routed Queries +### Simple routed Queries -## Query +#### Query ```sql SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Route", @@ -36,15 +35,14 @@ SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS "Table": "inventory, products" } ``` +--- -## Query - +#### Query ```sql SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` BETWEEN :1 /* INT64 */ AND :2 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Route", @@ -58,17 +56,17 @@ SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS "Table": "inventory, products" } ``` +--- -# Complex routed Queries +### Complex routed Queries -## Query +#### Query ```sql SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Limit", @@ -89,15 +87,14 @@ SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN ] } ``` +--- -## Query - +#### Query ```sql SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`created_at` BETWEEN :1 /* VARCHAR */ AND :2 /* VARCHAR */ GROUP BY `u`.`id` HAVING sum(`o`.`total_amount`) > :_total_spent /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -170,15 +167,14 @@ SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS ] } ``` +--- -## Query - +#### Query ```sql SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` ``` -## Plan - +#### Plan ```json { "OperatorType": "Aggregate", @@ -317,15 +313,14 @@ SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOI ] } ``` +--- -## Query - +#### Query ```sql SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Limit", @@ -442,15 +437,14 @@ SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `ca ] } ``` +--- -## Query - +#### Query ```sql SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS `s` ON `o`.`id` = `s`.`order_id` WHERE `s`.`shipped_date` IS NULL AND `o`.`created_at` < DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -493,15 +487,14 @@ SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS ] } ``` +--- -## Query - +#### Query ```sql SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` ``` -## Plan - +#### Plan ```json { "OperatorType": "Projection", @@ -570,15 +563,14 @@ SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM ` ] } ``` +--- -## Query - +#### Query ```sql SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) ``` -## Plan - +#### Plan ```json { "OperatorType": "Aggregate", @@ -602,15 +594,14 @@ SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `o ] } ``` +--- -## Query - +#### Query ```sql SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -640,15 +631,14 @@ SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` ] } ``` +--- -## Query - +#### Query ```sql SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL ``` -## Plan - +#### Plan ```json { "OperatorType": "Filter", @@ -691,15 +681,14 @@ SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON ] } ``` +--- -## Query - +#### Query ```sql SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` JOIN `reviews` AS `r` ON `u`.`id` = `r`.`user_id` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) AND `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ month) ``` -## Plan - +#### Plan ```json { "OperatorType": "Join", @@ -761,15 +750,14 @@ SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`. ] } ``` +--- -## Query - +#### Query ```sql SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` WHERE `r`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ week) GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :2 /* INT64 */ ``` -## Plan - +#### Plan ```json { "OperatorType": "Limit", @@ -790,4 +778,745 @@ SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN ] } ``` +--- + +## Top Queries +|Query ID|Usage Count|Total Query Time (ms)|Avg Query Time (ms)|Total Rows Examined| +|---|---|---|---|---| +|Q1|3|0.58|0.19|17,000| +|Q2|3|0.61|0.20|30,000| +|Q3|2|0.49|0.25|16,000| +|Q4|2|0.33|0.17|6,000| +|Q5|2|0.21|0.11|3,000| +|Q6|2|0.31|0.16|15,000| +|Q7|2|0.34|0.17|8,500| +|Q8|2|0.40|0.20|20,000| +|Q9|2|0.37|0.19|16,000| +|Q10|2|0.37|0.19|16,000| + +### Query Details +#### Q1 (`Complex routed`) +```sql +SELECT `u`.`username`, sum(`o`.`total_amount`) AS `total_spent` FROM `users` AS `u` JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`created_at` BETWEEN :1 /* VARCHAR */ AND :2 /* VARCHAR */ GROUP BY `u`.`id` HAVING sum(`o`.`total_amount`) > :_total_spent /* INT64 */ +``` +```json +{ + "OperatorType": "Filter", + "Predicate": "sum(o.total_amount) \u003e :_total_spent", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "any_value(0) AS username, sum(1) AS total_spent", + "GroupBy": "(2|3)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as username", + "sum(o.total_amount) * count(*) as total_spent", + ":3 as id", + ":4 as weight_string(u.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "o_user_id": 1 + }, + "TableName": "orders_users", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(o.total_amount) as total_spent, o.user_id from orders as o where 1 != 1 group by o.user_id", + "Query": "select sum(o.total_amount) as total_spent, o.user_id from orders as o where o.created_at between :1 and :2 group by o.user_id", + "Table": "orders" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select u.username, count(*), u.id, weight_string(u.id) from users as u where 1 != 1 group by u.id, weight_string(u.id)", + "Query": "select u.username, count(*), u.id, weight_string(u.id) from users as u where u.id = :o_user_id group by u.id, weight_string(u.id)", + "Table": "users", + "Values": [ + ":o_user_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } + ] + } +``` + +#### Q2 (`Complex routed`) +```sql +SELECT `m`.`sender_id`, COUNT(DISTINCT `m`.`receiver_id`) AS `unique_receivers` FROM `messages` AS `m` GROUP BY `m`.`sender_id` HAVING COUNT(DISTINCT `m`.`receiver_id`) > :_unique_receivers /* INT64 */ +``` +```json +{ + "OperatorType": "Filter", + "Predicate": "count(distinct m.receiver_id) \u003e :_unique_receivers", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "count_distinct(1|3) AS unique_receivers", + "GroupBy": "(0|2)", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id) from messages as m where 1 != 1 group by m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id)", + "OrderBy": "(0|2) ASC, (1|3) ASC", + "Query": "select m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id) from messages as m group by m.sender_id, m.receiver_id, weight_string(m.sender_id), weight_string(m.receiver_id) order by m.sender_id asc, m.receiver_id asc", + "Table": "messages" + } + ] + } + ] + } +``` + +#### Q3 (`Complex routed`) +```sql +SELECT `u`.`id`, `u`.`username` FROM `users` AS `u` LEFT JOIN `orders` AS `o` ON `u`.`id` = `o`.`user_id` WHERE `o`.`id` IS NULL +``` +```json +{ + "OperatorType": "Filter", + "Predicate": "o.id is null", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "LeftJoin", + "JoinColumnIndexes": "L:0,L:1,R:0", + "JoinVars": { + "u_id": 0 + }, + "TableName": "users_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select u.id, u.username from users as u where 1 != 1", + "Query": "select u.id, u.username from users as u", + "Table": "users" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select o.id from orders as o where 1 != 1", + "Query": "select o.id from orders as o where o.user_id = :u_id", + "Table": "orders" + } + ] + } + ] + } +``` + +#### Q4 (`Complex routed`) +```sql +SELECT `p`.`payment_method`, avg(`o`.`total_amount`) AS `avg_order_value` FROM `payments` AS `p` JOIN `orders` AS `o` ON `p`.`order_id` = `o`.`id` GROUP BY `p`.`payment_method` +``` +```json +{ + "OperatorType": "Projection", + "Expressions": [ + ":0 as payment_method", + "sum(o.total_amount) / count(o.total_amount) as avg_order_value" + ], + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum(1) AS avg_order_value, sum_count(2) AS count(o.total_amount)", + "GroupBy": "(0|3)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":3 as payment_method", + "count(*) * sum(o.total_amount) as avg_order_value", + "count(*) * count(o.total_amount) as count(o.total_amount)", + ":4 as weight_string(p.payment_method)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,L:1,L:3", + "JoinVars": { + "p_order_id": 2 + }, + "TableName": "payments_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), p.payment_method, p.order_id, weight_string(p.payment_method) from payments as p where 1 != 1 group by p.payment_method, p.order_id, weight_string(p.payment_method)", + "OrderBy": "(1|3) ASC", + "Query": "select count(*), p.payment_method, p.order_id, weight_string(p.payment_method) from payments as p group by p.payment_method, p.order_id, weight_string(p.payment_method) order by p.payment_method asc", + "Table": "payments" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(o.total_amount) as avg_order_value, count(o.total_amount) from orders as o where 1 != 1 group by .0", + "Query": "select sum(o.total_amount) as avg_order_value, count(o.total_amount) from orders as o where o.id = :p_order_id group by .0", + "Table": "orders", + "Values": [ + ":p_order_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } +``` + +#### Q5 (`Complex routed`) +```sql +SELECT `p`.`name`, avg(`r`.`rating`) AS `avg_rating` FROM `products` AS `p` JOIN `reviews` AS `r` ON `p`.`id` = `r`.`product_id` GROUP BY `p`.`id` ORDER BY avg(`r`.`rating`) DESC LIMIT :1 /* INT64 */ +``` +```json +{ + "OperatorType": "Limit", + "Count": "_vt_column_1", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select p.`name`, avg(r.rating) as avg_rating from products as p, reviews as r where 1 != 1 group by p.id", + "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci", + "Query": "select p.`name`, avg(r.rating) as avg_rating from products as p, reviews as r where p.id = r.product_id group by p.id order by avg(r.rating) desc limit :1", + "Table": "products, reviews" + } + ] + } +``` + +#### Q6 (`Simple routed`) +```sql +SELECT `p`.`name`, `i`.`stock_level` FROM `products` AS `p` JOIN `inventory` AS `i` ON `p`.`id` = `i`.`product_id` WHERE `i`.`stock_level` < :_i_stock_level /* INT64 */ +``` +```json +{ + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select p.`name`, i.stock_level from products as p, inventory as i where 1 != 1", + "Query": "select p.`name`, i.stock_level from products as p, inventory as i where i.stock_level \u003c :_i_stock_level and p.id = i.product_id", + "Table": "inventory, products" + } +``` + +#### Q7 (`Complex routed`) +```sql +SELECT `o`.`id`, `o`.`created_at` FROM `orders` AS `o` LEFT JOIN `shipments` AS `s` ON `o`.`id` = `s`.`order_id` WHERE `s`.`shipped_date` IS NULL AND `o`.`created_at` < DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) +``` +```json +{ + "OperatorType": "Filter", + "Predicate": "s.shipped_date is null", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "LeftJoin", + "JoinColumnIndexes": "L:0,L:1,R:0", + "JoinVars": { + "o_id": 0 + }, + "TableName": "orders_shipments", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select o.id, o.created_at from orders as o where 1 != 1", + "Query": "select o.id, o.created_at from orders as o where o.created_at \u003c date_sub(now(), interval :1 day)", + "Table": "orders" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select s.shipped_date from shipments as s where 1 != 1", + "Query": "select s.shipped_date from shipments as s where s.order_id = :o_id", + "Table": "shipments" + } + ] + } + ] + } +``` + +#### Q8 (`Complex routed`) +```sql +SELECT `c`.`name`, sum(`oi`.`price` * `oi`.`quantity`) AS `total_sales` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` GROUP BY `c`.`id` ORDER BY sum(`oi`.`price` * `oi`.`quantity`) DESC LIMIT :1 /* INT64 */ +``` +```json +{ + "OperatorType": "Limit", + "Count": "_vt_column_1", + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "1 DESC COLLATE utf8mb4_0900_ai_ci", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "any_value(0) AS name, sum(1) AS total_sales", + "GroupBy": "(2|3)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "sum(oi.price * oi.quantity) * count(*) as total_sales", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "oi_product_id": 1 + }, + "TableName": "order_items_products_categories", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select sum(oi.price * oi.quantity) as total_sales, oi.product_id from order_items as oi where 1 != 1 group by oi.product_id", + "Query": "select sum(oi.price * oi.quantity) as total_sales, oi.product_id from order_items as oi group by oi.product_id", + "Table": "order_items" + }, + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "count(*) * count(*) as count(*)", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "p_category_id": 1 + }, + "TableName": "products_categories", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), p.category_id from products as p where 1 != 1 group by p.category_id", + "Query": "select count(*), p.category_id from products as p where p.id = :oi_product_id group by p.category_id", + "Table": "products", + "Values": [ + ":oi_product_id" + ], + "Vindex": "xxhash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where 1 != 1 group by c.id, weight_string(c.id)", + "Query": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where c.id = :p_category_id group by c.id, weight_string(c.id)", + "Table": "categories", + "Values": [ + ":p_category_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } +``` + +#### Q9 (`Complex routed`) +```sql +SELECT `c`.`name`, COUNT(`o`.`id`) AS `order_count` FROM `categories` AS `c` JOIN `products` AS `p` ON `c`.`id` = `p`.`category_id` JOIN `order_items` AS `oi` ON `p`.`id` = `oi`.`product_id` JOIN `orders` AS `o` ON `oi`.`order_id` = `o`.`id` GROUP BY `c`.`id` +``` +```json +{ + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "any_value(0) AS name, sum_count(1) AS order_count", + "GroupBy": "(2|3)", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "count(o.id) * count(*) as order_count", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Sort", + "Variant": "Memory", + "OrderBy": "(3|4) ASC", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "oi_product_id": 1 + }, + "TableName": "order_items_orders_products_categories", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + "count(*) * count(o.id) as order_count", + ":2 as product_id" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,L:1", + "JoinVars": { + "oi_order_id": 2 + }, + "TableName": "order_items_orders", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), oi.product_id, oi.order_id from order_items as oi where 1 != 1 group by oi.product_id, oi.order_id", + "Query": "select count(*), oi.product_id, oi.order_id from order_items as oi group by oi.product_id, oi.order_id", + "Table": "order_items" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(o.id) as order_count from orders as o where 1 != 1 group by .0", + "Query": "select count(o.id) as order_count from orders as o where o.id = :oi_order_id group by .0", + "Table": "orders", + "Values": [ + ":oi_order_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + }, + { + "OperatorType": "Projection", + "Expressions": [ + ":0 as name", + "count(*) * count(*) as count(*)", + ":3 as id", + ":4 as weight_string(c.id)" + ], + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "Join", + "JoinColumnIndexes": "R:0,L:0,R:1,R:2,R:3", + "JoinVars": { + "p_category_id": 1 + }, + "TableName": "products_categories", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select count(*), p.category_id from products as p where 1 != 1 group by p.category_id", + "Query": "select count(*), p.category_id from products as p where p.id = :oi_product_id group by p.category_id", + "Table": "products", + "Values": [ + ":oi_product_id" + ], + "Vindex": "xxhash" + }, + { + "OperatorType": "Route", + "Variant": "EqualUnique", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where 1 != 1 group by c.id, weight_string(c.id)", + "Query": "select c.`name`, count(*), c.id, weight_string(c.id) from categories as c where c.id = :p_category_id group by c.id, weight_string(c.id)", + "Table": "categories", + "Values": [ + ":p_category_id" + ], + "Vindex": "xxhash" + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } +``` + +#### Q10 (`Complex routed`) +```sql +SELECT DATE(`o`.`created_at`) AS `order_date`, count(*) AS `order_count` FROM `orders` AS `o` WHERE `o`.`created_at` >= DATE_SUB(now(), INTERVAL :1 /* INT64 */ day) GROUP BY DATE(`o`.`created_at`) +``` +```json +{ + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "sum_count_star(1) AS order_count", + "GroupBy": "(0|2)", + "ResultColumns": 2, + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "main", + "Sharded": true + }, + "FieldQuery": "select DATE(o.created_at) as order_date, count(*) as order_count, weight_string(DATE(o.created_at)) from orders as o where 1 != 1 group by DATE(o.created_at), weight_string(DATE(o.created_at))", + "OrderBy": "(0|2) ASC", + "Query": "select DATE(o.created_at) as order_date, count(*) as order_count, weight_string(DATE(o.created_at)) from orders as o where o.created_at \u003e= date_sub(now(), interval :1 day) group by DATE(o.created_at), weight_string(DATE(o.created_at)) order by DATE(o.created_at) asc", + "Table": "orders" + } + ] + } +``` + +## Tables +|Table Name|Reads|Writes| +|---|---|---| +|orders|14|0| +|products|10|0| +|users|6|0| +|categories|4|0| +|order_items|4|0| +|reviews|4|0| +|inventory|3|0| +|messages|3|0| +|payments|2|0| +|shipments|2|0| + +### Column Usage +#### Table: `orders` (14 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|created_at|WHERE RANGE|57%| +|id|JOIN|43%| +|user_id|JOIN|43%| + +#### Table: `products` (10 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|id|JOIN|100%| +||GROUP|30%| +|category_id|JOIN|40%| + +#### Table: `users` (6 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|id|JOIN|100%| +||GROUP|50%| + +#### Table: `categories` (4 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|id|JOIN|100%| +||GROUP|100%| + +#### Table: `order_items` (4 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|product_id|JOIN|100%| +|order_id|JOIN|50%| + +#### Table: `reviews` (4 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|product_id|JOIN|75%| +|created_at|WHERE RANGE|50%| +|user_id|JOIN|25%| + +#### Table: `inventory` (3 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|product_id|JOIN|100%| +|stock_level|WHERE RANGE|100%| + +#### Table: `messages` (3 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|sender_id|GROUP|100%| + +#### Table: `payments` (2 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|order_id|JOIN|100%| +|payment_method|GROUP|100%| + +#### Table: `shipments` (2 reads and 0 writes) +|Column|Position|Used %| +|---|---|---| +|order_id|JOIN|100%| + +## Tables Joined +``` +orders ↔ users (Occurrences: 3) +└─ orders.user_id = users.id + +categories ↔ products (Occurrences: 2) +└─ categories.id = products.category_id + +inventory ↔ products (Occurrences: 2) +└─ inventory.product_id = products.id + +order_items ↔ products (Occurrences: 2) +└─ order_items.product_id = products.id + +products ↔ reviews (Occurrences: 2) +└─ products.id = reviews.product_id + +order_items ↔ orders (Occurrences: 1) +└─ order_items.order_id = orders.id + +orders ↔ payments (Occurrences: 1) +└─ orders.id = payments.order_id + +orders ↔ shipments (Occurrences: 1) +└─ orders.id = shipments.order_id + +reviews ↔ users (Occurrences: 1) +└─ reviews.user_id = users.id + +``` +## Failures +|Error|Count| +|---|---| +|syntax error at position 2|1| +|syntax error at position 14 near 'timestamp'|1| diff --git a/go/web/state/state.go b/go/web/state/state.go new file mode 100644 index 0000000..b0a1647 --- /dev/null +++ b/go/web/state/state.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state + +import ( + "fmt" + "net/http" + "sync" + "time" +) + +type State struct { + mu sync.Mutex + + port int64 + started bool +} + +func NewState(port int64) *State { + return &State{ + port: port, + } +} + +func (s *State) SetStarted(v bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.started = v +} + +func (s *State) Started() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.started +} + +func (s *State) GetPort() int64 { + s.mu.Lock() + defer s.mu.Unlock() + return s.port +} + +func (s *State) WaitUntilAvailable(timeout time.Duration) bool { + for { + select { + case <-time.After(timeout): + return false + case <-time.After(100 * time.Millisecond): + r, err := http.Get(fmt.Sprintf("http://localhost:%d/", s.port)) + if err != nil { + continue + } + if r.StatusCode == http.StatusOK { + return true + } + } + } +} diff --git a/go/web/templates/summarize.html b/go/web/templates/summarize.html index b31d7e0..00a44bd 100644 --- a/go/web/templates/summarize.html +++ b/go/web/templates/summarize.html @@ -13,6 +13,81 @@

Query Analysis Report

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Plan ComplexityCount
Pass-through{{.PlanAnalysis.PassThrough}}
Simple routed{{.PlanAnalysis.SimpleRouted}}
Complex routed{{.PlanAnalysis.Complex}}
Unplannable{{.PlanAnalysis.Unplannable}}
Total{{.PlanAnalysis.Total}}
+
+ + +
+

Simple Routed Queries

+ {{range $index, $query := .PlanAnalysis.SimpleRoutedQ}} + + + + + + + + + + + +
Query
{{$query.QueryStructure}}
+ 
+        {{- jsonToString $query.PlanOutput}}
+
+ {{end}} + +

Complex Queries

+ {{range $index, $query := .PlanAnalysis.ComplexQ}} + + + + + + + + + + + +
Query
{{$query.QueryStructure}}
+ 
+        {{- jsonToString $query.PlanOutput}}
+
+ {{end}} +
+
@@ -29,10 +104,10 @@

Query Analysis Report

{{range $index, $query := .HotQueries}} - - - - + + + + {{end}} @@ -52,7 +127,7 @@

Query Analysis Report

{{range $index, $query := .HotQueries}} - + {{end}} diff --git a/go/web/web.go b/go/web/web.go index d765ca1..c52af58 100644 --- a/go/web/web.go +++ b/go/web/web.go @@ -6,7 +6,6 @@ import ( "fmt" "html/template" "io" - "log" "net/http" "os" "time" @@ -14,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/vitessio/vt/go/summarize" + "github.com/vitessio/vt/go/web/state" ) func RenderFileToGin(fileName string, data any, c *gin.Context) { @@ -48,6 +48,11 @@ type SummaryOutput struct { func getFuncMap() template.FuncMap { return template.FuncMap{ + "jsonToString": func(j json.RawMessage) string { + var formattedJSON bytes.Buffer + _ = json.Indent(&formattedJSON, j, "", " ") + return formattedJSON.String() + }, "add": func(a, b int) int { return a + b }, "divide": func(a, b any) float64 { if b == 0 || b == nil { @@ -84,7 +89,7 @@ func addFuncMap(r *gin.Engine) { r.SetFuncMap(getFuncMap()) } -func Run(port int64) { +func Run(s *state.State) error { gin.SetMode(gin.ReleaseMode) gin.DefaultWriter = io.Discard // Disable logging r := gin.Default() @@ -125,10 +130,12 @@ func Run(port int64) { RenderFileToGin("summarize.html", &summarizeOutput, c) }) - if os.WriteFile("/dev/stderr", []byte(fmt.Sprintf("Starting web server on http://localhost:%d\n", port)), 0o600) != nil { - panic("Failed to write to /dev/stderr") + if _, err := fmt.Fprintf(os.Stderr, "Starting web server on http://localhost:%d\n", s.GetPort()); err != nil { + return err } - if err := r.Run(fmt.Sprintf(":%d", port)); err != nil { - log.Fatalf("Failed to start server: %v", err) + + if err := r.Run(fmt.Sprintf(":%d", s.GetPort())); err != nil { + return err } + return nil }
{{$index | add 1}} {{$query.UsageCount}}{{$query.QueryTime}}{{divide .QueryTime .UsageCount}}{{$query.RowsExamined}}{{$query.QueryAnalysisResult.UsageCount}}{{$query.QueryAnalysisResult.QueryTime}}{{divide $query.QueryAnalysisResult.QueryTime $query.QueryAnalysisResult.UsageCount}}{{$query.QueryAnalysisResult.RowsExamined}}
{{$index | add 1}}{{.QueryStructure}}{{$query.QueryAnalysisResult.QueryStructure}}