WIP: add synchronous execution of file tests and panic on file err

Author: dylanhitt

go.mod298498081.tmp

@@ -52,8 +52,10 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
@@ -112,6 +114,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU=
 golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
@@ -123,6 +126,7 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=

go.mod427131847.tmp

@@ -13,7 +13,6 @@ const (
 type AddCommandContext struct {
 	Verbose    bool
 	NoColor    bool
-	FileOrder  bool
 	Concurrent int
 }
 
@@ -22,7 +21,6 @@ func NewAddContextFromCli(c *cli.Context) AddCommandContext {
 	return AddCommandContext{
 		Verbose:    c.Bool("verbose"),
 		NoColor:    c.Bool("no-color"),
-		FileOrder:  c.Bool("file-order"),
 		Concurrent: c.Int("concurrent"),
 	}
 }

go.mod939984059.tmp

@@ -33,6 +33,7 @@ func TestCommand(input string, title string, ctx AddCommandContext) error {
 	if inputType.IsDir() {
 		fmt.Println("Starting test against directory: " + input + "...")
 		fmt.Println("")
+		// since no handling of file errors occur should we not allow title
 		results, err = testDir(input, title)
 	} else {
 		fmt.Println("Starting test file " + input + "...")
@@ -44,7 +45,7 @@ func TestCommand(input string, title string, ctx AddCommandContext) error {
 		return fmt.Errorf(err.Error())
 	}
 
-	out := output.NewCliOutput(!ctx.NoColor, ctx.FileOrder)
+	out := output.NewCliOutput(!ctx.NoColor)
 	if !out.Start(results) {
 		return fmt.Errorf("Test suite failed, use --verbose for more detailed output")
 	}
@@ -61,24 +62,26 @@ func testDir(directory string, title string) (<-chan runtime.TestResult, error)
 	results := make(chan runtime.TestResult)
 	var wg sync.WaitGroup
 
-	for _, f := range files {
-		wg.Add(1)
-
-		go func(f os.FileInfo) {
-			defer wg.Done()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for _, f := range files {
+			// Skip reading dirs for now. Should we also check valid file types?
+			if f.IsDir() {
+				continue
+			}
 
 			fileResults, err := testFile(directory+"/"+f.Name(), title)
 			if err != nil {
-				results <- runtime.TestResult{FileError: err, FileName: f.Name()}
-				return
+				panic(fmt.Sprintf("%s: %s", f.Name(), err))
 			}
 
 			for r := range fileResults {
 				r.FileName = f.Name()
-				results <- r //fan in results
+				results <- r
 			}
-		}(f)
-	}
+		}
+	}()
 
 	go func(ch chan runtime.TestResult) {
 		wg.Wait()

go.sum

@@ -56,16 +56,6 @@ func Test_TestCommand_Dir(t *testing.T) {
 	}
 }
 
-func Test_TestCommand_Dir_With_Title(t *testing.T) {
-	err := TestCommand("../../examples", "it should fail", AddCommandContext{})
-
-	if runtime.GOOS == "windows" {
-		assert.Contains(t, err.Error(), "Test suite failed, use --verbose for more detailed output")
-	} else {
-		assert.Equal(t, "Test suite failed, use --verbose for more detailed output", err.Error())
-	}
-}
-
 func captureOutput(f func()) string {
 	reader, writer, err := os.Pipe()
 	if err != nil {

pkg/app/app.go

@@ -6,11 +6,9 @@ import (
 	"log"
 	"os"
 	run "runtime"
-	"sort"
 	"time"
 
 	"github.com/SimonBaeumer/commander/pkg/runtime"
-	"github.com/SimonBaeumer/commander/pkg/suite"
 	"github.com/logrusorgru/aurora"
 )
 
@@ -20,15 +18,13 @@ var au aurora.Aurora
 type OutputWriter struct {
 	out   io.Writer
 	color bool
-	order bool
 }
 
 // NewCliOutput creates a new OutputWriter with a stdout writer
-func NewCliOutput(color bool, order bool) OutputWriter {
+func NewCliOutput(color bool) OutputWriter {
 	return OutputWriter{
 		out:   os.Stdout,
 		color: color,
-		order: order,
 	}
 }
 
@@ -39,36 +35,17 @@ func (w *OutputWriter) Start(results <-chan runtime.TestResult) bool {
 		au = aurora.NewAurora(false)
 	}
 
-	var failed int
-	var fileErrors []error
+	failed := 0
 	var testResults []runtime.TestResult
 	start := time.Now()
 
 	for r := range results {
-		if r.FileError != nil {
-			//only append FileErrors that are not title Errors
-			if _, ok := r.FileError.(*suite.TitleErr); !ok {
-				fileErrors = append(fileErrors, fmt.Errorf("[%s] %s", r.FileName, r.FileError))
-			}
-			continue
-		}
 		testResults = append(testResults, r)
-		if !w.order { //if no order print now
-			failed += w.readResult(r)
-		}
+		failed += w.printResult(r)
 	}
 
 	duration := time.Since(start)
 
-	if w.order { //Maintain file order
-		sort.SliceStable(testResults, func(i, j int) bool {
-			return testResults[i].FileName < testResults[j].FileName
-		})
-		for _, r := range testResults {
-			failed += w.readResult(r)
-		}
-	}
-
 	if failed > 0 {
 		w.printFailures(testResults)
 	}
@@ -82,12 +59,10 @@ func (w *OutputWriter) Start(results <-chan runtime.TestResult) bool {
 		w.fprintf(au.Green(summary))
 	}
 
-	w.printFileErrors(fileErrors)
-
 	return failed == 0
 }
 
-func (w *OutputWriter) readResult(r runtime.TestResult) int {
+func (w *OutputWriter) printResult(r runtime.TestResult) int {
 	if !r.ValidationResult.Success {
 		str := fmt.Sprintf("✗ [%s] %s", r.Node, r.TestCase.Title)
 		str = w.addFile(str, r)
@@ -140,18 +115,6 @@ func (w *OutputWriter) addTries(s string, r runtime.TestResult) string {
 	return s
 }
 
-func (w *OutputWriter) printFileErrors(errors []error) {
-	if len(errors) <= 0 {
-		return
-	}
-
-	w.fprintf("")
-	w.fprintf("File Errors:")
-	for _, e := range errors {
-		w.fprintf(e)
-	}
-}
-
 func (w *OutputWriter) fprintf(a ...interface{}) {
 	if _, err := fmt.Fprintln(w.out, a...); err != nil {
 		log.Fatal(err)

pkg/app/test_command.go

@@ -12,7 +12,7 @@ import (
 )
 
 func Test_NewCliOutput(t *testing.T) {
-	got := NewCliOutput(true, true)
+	got := NewCliOutput(true)
 	assert.IsType(t, OutputWriter{}, got)
 }
 
@@ -25,7 +25,7 @@ func Test_Start(t *testing.T) {
 	go func() {
 		defer wg.Done()
 
-		writer := OutputWriter{out: &buf, order: true}
+		writer := OutputWriter{out: &buf}
 		got := writer.Start(results)
 
 		assert.False(t, got)
@@ -94,11 +94,6 @@ func Test_Start(t *testing.T) {
 		Tries: 2,
 	}
 
-	results <- runtime.TestResult{
-		FileName:  "MySweetFile",
-		FileError: fmt.Errorf("Some file error message"),
-	}
-
 	close(results)
 	wg.Wait()
 
@@ -109,7 +104,6 @@ func Test_Start(t *testing.T) {
 	assert.Contains(t, output, "✗ [local] 'Invalid command' could not be executed with error message")
 	assert.Contains(t, output, "✗ [ssh-host1] 'Failed test on stderr', on property 'Stderr'")
 	assert.Contains(t, output, "Some error message")
-	assert.Contains(t, output, "Some file error message")
 }
 
 func Test_SuccessSuite(t *testing.T) {

pkg/app/test_command_test.go

@@ -134,7 +134,6 @@ type TestResult struct {
 	Node             string
 	Error            error
 	FileName         string
-	FileError        error
 }
 
 // Start starts the given test suite and executes all tests