Add --dir flag, add output messages to describe input errors

Author: dylanhitt

cmd/commander/commander.go

@@ -60,9 +60,8 @@ func createTestCommand() cli.Command {
 				EnvVar: "COMMANDER_VERBOSE",
 			},
 			cli.BoolFlag{
-				Name:   "file-order",
-				Usage:  "Preserve file order",
-				EnvVar: "COMMANDER_FILE_ORDER",
+				Name:  "dir",
+				Usage: "Execute all test files in directory - e.g. /path/to/test_files/",
 			},
 		},
 		Action: func(c *cli.Context) error {

commander_linux.yaml

@@ -17,7 +17,7 @@ tests:
     exit-code: 0
 
   test directory:
-    command: ./commander test integration/linux/
+    command: ./commander test --dir integration/linux/
     stdout:
       contains:
         - ✓ [nodes.yaml] [ssh-host] it should test ssh host
@@ -27,4 +27,11 @@ tests:
         - ✓ [nodes.yaml] [ssh-host-default] it should test multiple hosts
         - ✓ [nodes.yaml] [local] it should test multiple hosts
         - ✓ [docker.yaml] [docker-host] cat /etc/os-release
-    exit-code: 0
+    exit-code: 0
+
+  test file with directory:
+    command: ./commander test integration/linux/
+    stdout:
+      contains:
+        - Use --dir to test directories with multiple test files
+    exit-code: 1

commander_unix.yaml

@@ -67,8 +67,15 @@ tests:
     exit-code: 1
 
   test directory order:
-    command: ./commander test --file-order integration/unix/directory_test/
+    command: ./commander test --dir integration/unix/directory_test/
     stdout: 
       lines:
         3: ✓ [alpha_test.yaml] [local] sleep test
-        4: ✓ [beta_test.yaml] [local] ehco hello
+        4: ✓ [beta_test.yaml] [local] ehco hello
+  
+  test missing dir flag:
+    command: ./commander test integration/unix/directory_test/
+    stdout: 
+      contains:
+        - 'integration/unix/directory_test/: is a directory'
+    exit-code: 1

integration/unix/directory_test/place_holder/test

@@ -0,0 +1,4 @@
+Hello, I am file in a directory. Don't delete me
+as I am a place hold for commander's integration tests.
+Please see missing test 'test missing dir flag' in
+commander_unix.yaml for why (:

pkg/app/app.go

@@ -13,6 +13,7 @@ const (
 type AddCommandContext struct {
 	Verbose    bool
 	NoColor    bool
+	Dir        bool
 	Concurrent int
 }
 
@@ -21,6 +22,7 @@ func NewAddContextFromCli(c *cli.Context) AddCommandContext {
 	return AddCommandContext{
 		Verbose:    c.Bool("verbose"),
 		NoColor:    c.Bool("no-color"),
+		Dir:        c.Bool("dir"),
 		Concurrent: c.Int("concurrent"),
 	}
 }

pkg/app/test_command.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/SimonBaeumer/commander/pkg/output"
@@ -13,32 +14,32 @@ import (
 )
 
 // TestCommand executes the test argument
-// file is the path to the configuration file
-// title ist the title of test which should be executed, if empty it will execute all tests
+// testPath is the path to the configuration object, an object can be a dir or file
+// titleFilterTitle is the title of test which should be executed, if empty it will execute all tests
 // ctx holds the command flags
-func TestCommand(input string, title string, ctx AddCommandContext) error {
+// when --dir is enabled testFilterPath must be a zero value
+func TestCommand(testPath string, testFilterTitle string, ctx AddCommandContext) error {
 	if ctx.Verbose == true {
 		log.SetOutput(os.Stdout)
 	}
 
-	if input == "" {
-		input = CommanderFile
-	}
-	inputType, err := os.Stat(input)
-	if err != nil {
-		return fmt.Errorf("Error " + err.Error())
+	if testPath == "" {
+		testPath = CommanderFile
 	}
 
 	var results <-chan runtime.TestResult
-	if inputType.IsDir() {
-		fmt.Println("Starting test against directory: " + input + "...")
+	var err error
+	if ctx.Dir {
+		if testFilterTitle != "" {
+			return fmt.Errorf("Test may not be filtered when --dir is enabled")
+		}
+		fmt.Println("Starting test against directory: " + testPath + "...")
 		fmt.Println("")
-		// since no handling of file errors occur should we not allow title
-		results, err = testDir(input, title)
+		results, err = testDir(testPath, testFilterTitle)
 	} else {
-		fmt.Println("Starting test file " + input + "...")
+		fmt.Println("Starting test file " + testPath + "...")
 		fmt.Println("")
-		results, err = testFile(input, title)
+		results, err = testFile(testPath, testFilterTitle)
 	}
 
 	if err != nil {
@@ -56,7 +57,7 @@ func TestCommand(input string, title string, ctx AddCommandContext) error {
 func testDir(directory string, title string) (<-chan runtime.TestResult, error) {
 	files, err := ioutil.ReadDir(directory)
 	if err != nil {
-		return nil, fmt.Errorf("Error " + err.Error())
+		return nil, fmt.Errorf(err.Error())
 	}
 
 	results := make(chan runtime.TestResult)
@@ -94,6 +95,9 @@ func testDir(directory string, title string) (<-chan runtime.TestResult, error)
 func testFile(input string, title string) (<-chan runtime.TestResult, error) {
 	content, err := ioutil.ReadFile(input)
 	if err != nil {
+		if strings.Contains(err.Error(), "is a directory") {
+			return nil, fmt.Errorf("Error %s: is a directory\nUse --dir to test directories with multiple test files", input)
+		}
 		return nil, fmt.Errorf("Error " + err.Error())
 	}
 

pkg/app/test_command_test.go

@@ -30,32 +30,52 @@ func Test_TestCommand(t *testing.T) {
 	err := TestCommand("commander.yaml", "", AddCommandContext{})
 
 	if runtime.GOOS == "windows" {
-		assert.Contains(t, err.Error(), "Error CreateFile commander.yaml:")
+		assert.Contains(t, err.Error(), "Error open commander.yaml:")
 	} else {
-		assert.Equal(t, "Error stat commander.yaml: no such file or directory", err.Error())
+		assert.Equal(t, "Error open commander.yaml: no such file or directory", err.Error())
 	}
 }
 
 func Test_TestCommand_ShouldUseCustomFile(t *testing.T) {
 	err := TestCommand("my-test.yaml", "", AddCommandContext{})
 
 	if runtime.GOOS == "windows" {
-		assert.Contains(t, err.Error(), "Error CreateFile my-test.yaml: ")
+		assert.Contains(t, err.Error(), "Error open my-test.yaml:")
 	} else {
-		assert.Equal(t, "Error stat my-test.yaml: no such file or directory", err.Error())
+		assert.Equal(t, "Error open my-test.yaml: no such file or directory", err.Error())
 	}
 }
 
-func Test_TestCommand_Dir(t *testing.T) {
+func Test_TestCommand_File_WithDir(t *testing.T) {
 	err := TestCommand("../../examples", "", AddCommandContext{})
 
+	if runtime.GOOS == "windows" {
+		assert.Contains(t, err.Error(), "is a directory")
+	} else {
+		assert.Equal(t, "Error ../../examples: is a directory\nUse --dir to test directories with multiple test files", err.Error())
+	}
+}
+
+func Test_TestCommand_Dir(t *testing.T) {
+	err := TestCommand("../../examples", "", AddCommandContext{Dir: true})
+
 	if runtime.GOOS == "windows" {
 		assert.Contains(t, err.Error(), "Test suite failed, use --verbose for more detailed output")
 	} else {
 		assert.Equal(t, "Test suite failed, use --verbose for more detailed output", err.Error())
 	}
 }
 
+func Test_TestCommand_Dir_FilterTitle(t *testing.T) {
+	err := TestCommand("/fake", "hello", AddCommandContext{Dir: true})
+
+	if runtime.GOOS == "windows" {
+		assert.Contains(t, err.Error(), "Test may not be filtered when --dir is enabled")
+	} else {
+		assert.Equal(t, "Test may not be filtered when --dir is enabled", err.Error())
+	}
+}
+
 func captureOutput(f func()) string {
 	reader, writer, err := os.Pipe()
 	if err != nil {

pkg/output/cli.go

@@ -41,7 +41,12 @@ func (w *OutputWriter) Start(results <-chan runtime.TestResult) bool {
 
 	for r := range results {
 		testResults = append(testResults, r)
-		failed += w.printResult(r)
+		if r.ValidationResult.Success {
+			w.printResult(r, "✓")
+		} else {
+			w.printResult(r, "✗")
+			failed++
+		}
 	}
 
 	duration := time.Since(start)
@@ -62,19 +67,11 @@ func (w *OutputWriter) Start(results <-chan runtime.TestResult) bool {
 	return failed == 0
 }
 
-func (w *OutputWriter) printResult(r runtime.TestResult) int {
-	if !r.ValidationResult.Success {
-		str := fmt.Sprintf("✗ [%s] %s", r.Node, r.TestCase.Title)
-		str = w.addFile(str, r)
-		s := w.addTries(str, r)
-		w.fprintf(au.Red(s))
-		return 1
-	}
-	str := fmt.Sprintf("✓ [%s] %s", r.Node, r.TestCase.Title)
+func (w *OutputWriter) printResult(r runtime.TestResult, mark string) {
+	str := fmt.Sprintf("%s [%s] %s", mark, r.Node, r.TestCase.Title)
 	str = w.addFile(str, r)
 	s := w.addTries(str, r)
 	w.fprintf(s)
-	return 0
 }
 
 func (w *OutputWriter) printFailures(results []runtime.TestResult) {