Merge pull request #120 from dylanhitt/add-test-against-directory

SimonBaeumer · web-flow · commit d2b4251f92a4 · 2020-05-23T11:13:59.000+02:00
Add test against directory
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+# v2.1.0
+ 
+  - Added `--dir` flag to execute all commander test suites within a directory. This feature is not recursive.
+
 # v2.0.0
  
   - Added `nodes` which allow remote execution of tests
diff --git a/README.md b/README.md
@@ -195,6 +195,9 @@ $ ./commander test /tmp/test.yaml
 
 # Execute a single test
 $ ./commander test /tmp/test.yaml "my test"
+
+# Execute suites within a test directory
+$ ./commander test --dir /tmp
 ```
 
 ### Adding tests
diff --git a/cmd/commander/commander.go b/cmd/commander/commander.go
@@ -2,13 +2,14 @@ package main
 
 import (
 	"fmt"
-	"github.com/SimonBaeumer/commander/pkg/app"
-	"github.com/urfave/cli"
 	"io/ioutil"
 	"log"
 	"os"
 	"path"
 	"strings"
+
+	"github.com/SimonBaeumer/commander/pkg/app"
+	"github.com/urfave/cli"
 )
 
 var version string
@@ -58,6 +59,10 @@ func createTestCommand() cli.Command {
 				Usage:  "More output for debugging",
 				EnvVar: "COMMANDER_VERBOSE",
 			},
+			cli.BoolFlag{
+				Name:  "dir",
+				Usage: "Execute all test files in a directory sorted by file name, this is not recursive - e.g. /path/to/test_files/",
+			},
 		},
 		Action: func(c *cli.Context) error {
 			if c.Bool("verbose") {
diff --git a/commander_linux.yaml b/commander_linux.yaml
@@ -14,4 +14,24 @@ tests:
   test docker:
     command: ./commander test integration/linux/docker.yaml
     stdout: ✓ [docker-host] cat /etc/os-release
-    exit-code: 0
+    exit-code: 0
+
+  test directory:
+    command: ./commander test --dir integration/linux/
+    stdout:
+      contains:
+        - ✓ [nodes.yaml] [ssh-host] it should test ssh host
+        - ✓ [nodes.yaml] [ssh-host] it should set env variable
+        - ✓ [nodes.yaml] [ssh-host-default] it should be executed on ssh-host-default
+        - ✓ [nodes.yaml] [ssh-host] it should test multiple hosts
+        - ✓ [nodes.yaml] [ssh-host-default] it should test multiple hosts
+        - ✓ [nodes.yaml] [local] it should test multiple hosts
+        - ✓ [docker.yaml] [docker-host] cat /etc/os-release
+    exit-code: 0
+
+  test file with directory:
+    command: ./commander test integration/linux/
+    stdout:
+      contains:
+        - Use --dir to test directories with multiple test files
+    exit-code: 1
diff --git a/commander_unix.yaml b/commander_unix.yaml
@@ -65,3 +65,17 @@ tests:
         - ✓ [local] it should retry failed commands, retries 2
         - ✗ [local] it should retry failed commands with an interval, retries 2
     exit-code: 1
+
+  test directory order:
+    command: ./commander test --dir integration/unix/directory_test/
+    stdout: 
+      lines:
+        3: ✓ [alpha_test.yaml] [local] sleep test
+        4: ✓ [beta_test.yaml] [local] ehco hello
+  
+  test missing dir flag:
+    command: ./commander test integration/unix/directory_test/
+    stdout: 
+      contains:
+        - 'integration/unix/directory_test/: is a directory'
+    exit-code: 1
diff --git a/examples/commander.yaml b/examples/commander.yaml
@@ -18,5 +18,5 @@ tests:
 
   it should fail:
     command: invalid
-    stderr: "/bin/sh: 1: invalid: not found"
-    exit-code: 127
+    stderr: "/bin/sh: 1: command invalid: not found"
+    exit-code: 127
diff --git a/examples/environment.yaml b/examples/environment.yaml
@@ -12,4 +12,4 @@ tests:
     config:
       env:
         KEY: local
-    exit-code: 0
+    exit-code: 0
diff --git a/examples/minimal_test.yaml b/examples/minimal_test.yaml
@@ -1,3 +1,3 @@
 tests:
   echo hello:
-    exit-code: 0
+    exit-code: 0
diff --git a/integration/linux/docker.yaml b/integration/linux/docker.yaml
@@ -20,4 +20,4 @@ tests:
         BUG_REPORT_URL="https://bugs.alpinelinux.org/"
 
   "id -u":
-    stdout: "1001"
+    stdout: "1001"
diff --git a/integration/unix/directory_test/alpha_test.yaml b/integration/unix/directory_test/alpha_test.yaml
@@ -0,0 +1,4 @@
+tests:
+  sleep test:
+    command: sleep 2
+    exit-code: 0
diff --git a/integration/unix/directory_test/beta_test.yaml b/integration/unix/directory_test/beta_test.yaml
@@ -0,0 +1,7 @@
+tests:
+  ehco hello:
+    command: echo hello
+    stdout: 
+      contains: 
+        - hello
+    exit-code: 0
diff --git a/integration/unix/directory_test/place_holder/.gitkeep b/integration/unix/directory_test/place_holder/.gitkeep
@@ -0,0 +1,4 @@
+Hello, I am file in a directory. Don't delete me
+as I am a place hold for commander's integration tests.
+Please see the test 'test missing dir flag' in
+commander_unix.yaml for why (:
diff --git a/pkg/app/app.go b/pkg/app/app.go
@@ -13,6 +13,7 @@ const (
 type AddCommandContext struct {
 	Verbose    bool
 	NoColor    bool
+	Dir        bool
 	Concurrent int
 }
 
@@ -21,6 +22,7 @@ func NewAddContextFromCli(c *cli.Context) AddCommandContext {
 	return AddCommandContext{
 		Verbose:    c.Bool("verbose"),
 		NoColor:    c.Bool("no-color"),
+		Dir:        c.Bool("dir"),
 		Concurrent: c.Int("concurrent"),
 	}
 }
diff --git a/pkg/app/test_command.go b/pkg/app/test_command.go
@@ -2,32 +2,100 @@ package app
 
 import (
 	"fmt"
-	"github.com/SimonBaeumer/commander/pkg/output"
-	"github.com/SimonBaeumer/commander/pkg/runtime"
-	"github.com/SimonBaeumer/commander/pkg/suite"
 	"io/ioutil"
 	"log"
 	"os"
+	"path"
+	"sync"
+
+	"github.com/SimonBaeumer/commander/pkg/output"
+	"github.com/SimonBaeumer/commander/pkg/runtime"
+	"github.com/SimonBaeumer/commander/pkg/suite"
 )
 
 // TestCommand executes the test argument
-// file is the path to the configuration file
-// title ist the title of test which should be executed, if empty it will execute all tests
-// ctx holds the command flags
-func TestCommand(file string, title string, ctx AddCommandContext) error {
+// testPath is the path to the test suite config, it can be a dir or file
+// titleFilterTitle is the title of test which should be executed, if empty it will execute all tests
+// ctx holds the command flags. If directory scanning is enabled with --dir it is
+// not supported to filter tests, therefore testFilterTitle is an empty string
+func TestCommand(testPath string, testFilterTitle string, ctx AddCommandContext) error {
 	if ctx.Verbose == true {
 		log.SetOutput(os.Stdout)
 	}
 
-	if file == "" {
-		file = CommanderFile
+	if testPath == "" {
+		testPath = CommanderFile
+	}
+
+	var results <-chan runtime.TestResult
+	var err error
+	if ctx.Dir {
+		if testFilterTitle != "" {
+			return fmt.Errorf("Test may not be filtered when --dir is enabled")
+		}
+		fmt.Println("Starting test against directory: " + testPath + "...")
+		fmt.Println("")
+		results, err = testDir(testPath)
+	} else {
+		fmt.Println("Starting test file " + testPath + "...")
+		fmt.Println("")
+		results, err = testFile(testPath, testFilterTitle)
+	}
+
+	if err != nil {
+		return fmt.Errorf(err.Error())
+	}
+
+	out := output.NewCliOutput(!ctx.NoColor)
+	if !out.Start(results) {
+		return fmt.Errorf("Test suite failed, use --verbose for more detailed output")
+	}
+
+	return nil
+}
+
+func testDir(directory string) (<-chan runtime.TestResult, error) {
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, fmt.Errorf(err.Error())
 	}
 
-	fmt.Println("Starting test file " + file + "...")
-	fmt.Println("")
-	content, err := ioutil.ReadFile(file)
+	results := make(chan runtime.TestResult)
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for _, f := range files {
+			// Skip reading dirs for now. Should we also check valid file types?
+			if f.IsDir() {
+				continue
+			}
+
+			fileResults, err := testFile(path.Join(directory, f.Name()), "")
+			if err != nil {
+				panic(fmt.Sprintf("%s: %s", f.Name(), err))
+			}
+
+			for r := range fileResults {
+				r.FileName = f.Name()
+				results <- r
+			}
+		}
+	}()
+
+	go func(ch chan runtime.TestResult) {
+		wg.Wait()
+		close(results)
+	}(results)
+
+	return results, nil
+}
+
+func testFile(filePath string, title string) (<-chan runtime.TestResult, error) {
+	content, err := readFile(filePath)
 	if err != nil {
-		return fmt.Errorf("Error " + err.Error())
+		return nil, fmt.Errorf("Error " + err.Error())
 	}
 
 	var s suite.Suite
@@ -37,18 +105,31 @@ func TestCommand(file string, title string, ctx AddCommandContext) error {
 	if title != "" {
 		test, err := s.GetTestByTitle(title)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		tests = []runtime.TestCase{test}
 	}
 
 	r := runtime.NewRuntime(s.Nodes...)
-
 	results := r.Start(tests)
-	out := output.NewCliOutput(!ctx.NoColor)
-	if !out.Start(results) {
-		return fmt.Errorf("Test suite failed, use --verbose for more detailed output")
+
+	return results, nil
+}
+
+func readFile(filePath string) ([]byte, error) {
+	f, err := os.Stat(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("open %s: no such file or directory", filePath)
 	}
 
-	return nil
+	if f.IsDir() {
+		return nil, fmt.Errorf("%s: is a directory\nUse --dir to test directories with multiple test files", filePath)
+	}
+
+	content, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return content, nil
 }
diff --git a/pkg/app/test_command_test.go b/pkg/app/test_command_test.go
@@ -2,13 +2,14 @@ package app
 
 import (
 	"bytes"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"log"
 	"os"
 	"runtime"
 	"sync"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_TestCommand_Verbose(t *testing.T) {
@@ -39,12 +40,42 @@ func Test_TestCommand_ShouldUseCustomFile(t *testing.T) {
 	err := TestCommand("my-test.yaml", "", AddCommandContext{})
 
 	if runtime.GOOS == "windows" {
-		assert.Contains(t, err.Error(), "Error open my-test.yaml: ")
+		assert.Contains(t, err.Error(), "Error open my-test.yaml:")
 	} else {
 		assert.Equal(t, "Error open my-test.yaml: no such file or directory", err.Error())
 	}
 }
 
+func Test_TestCommand_File_WithDir(t *testing.T) {
+	err := TestCommand("../../examples", "", AddCommandContext{})
+
+	if runtime.GOOS == "windows" {
+		assert.Contains(t, err.Error(), "is a directory")
+	} else {
+		assert.Equal(t, "Error ../../examples: is a directory\nUse --dir to test directories with multiple test files", err.Error())
+	}
+}
+
+func Test_TestCommand_Dir(t *testing.T) {
+	err := TestCommand("../../examples", "", AddCommandContext{Dir: true})
+
+	if runtime.GOOS == "windows" {
+		assert.Contains(t, err.Error(), "Test suite failed, use --verbose for more detailed output")
+	} else {
+		assert.Equal(t, "Test suite failed, use --verbose for more detailed output", err.Error())
+	}
+}
+
+func Test_TestCommand_Dir_FilterTitle(t *testing.T) {
+	err := TestCommand("/fake", "hello", AddCommandContext{Dir: true})
+
+	if runtime.GOOS == "windows" {
+		assert.Contains(t, err.Error(), "Test may not be filtered when --dir is enabled")
+	} else {
+		assert.Equal(t, "Test may not be filtered when --dir is enabled", err.Error())
+	}
+}
+
 func captureOutput(f func()) string {
 	reader, writer, err := os.Pipe()
 	if err != nil {
diff --git a/pkg/output/cli.go b/pkg/output/cli.go
diff --git a/pkg/output/cli_test.go b/pkg/output/cli_test.go
diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go
diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go
diff --git a/pkg/suite/suite.go b/pkg/suite/suite.go
