Add unit tests for runtime

Author: SimonBaeumer

CHANGELOG.md

@@ -1,3 +1,9 @@
+# v2.0.0
+ 
+  - Added `nodes` which allow remote execution of tests
+  - Added `SSHExecutor` and `LocalExecutor`
+  - Removed `concurrent` argument from `test` command
+
 # v1.3.0
 
  - Added `xml` assertion to `stdout` and `stderr`

README.md

@@ -653,11 +653,11 @@ Enables ssh tests in unit test suite and sets the credentials for the target hos
 `COMMANDER_SSH_TEST` must be set to `1` to enable ssh tests.
 
 ```
-COMMANDER_TEST_SSH=1
-COMMANDER_TEST_SSH_HOST=localhost:2222
-COMMANDER_TEST_SSH_PASS=password
-COMMANDER_TEST_SSH_USER=root
-COMMANDER_TEST_SSH_IDENTITY_FILE=~/.ssh/id_rsa
+export COMMANDER_TEST_SSH=1
+export COMMANDER_TEST_SSH_HOST=localhost:2222
+export COMMANDER_TEST_SSH_PASS=pass
+export COMMANDER_TEST_SSH_USER=root
+export COMMANDER_TEST_SSH_IDENTITY_FILE=integration/containers/ssh/.ssh/id_rsa
 ```
 
 ## Misc

cmd/commander/commander.go

@@ -48,11 +48,6 @@ func createTestCommand() cli.Command {
 		Usage:     "Execute the test suite, by default it will use the commander.yaml from your current directory",
 		ArgsUsage: "[file] [title]",
 		Flags: []cli.Flag{
-			cli.IntFlag{
-				Name:   "concurrent",
-				EnvVar: "COMMANDER_CONCURRENT",
-				Usage:  "Set the max amount of tests which should run concurrently",
-			},
 			cli.BoolFlag{
 				Name:   "no-color",
 				EnvVar: "COMMANDER_NO_COLOR",

pkg/app/test_command.go

@@ -44,7 +44,7 @@ func TestCommand(file string, title string, ctx AddCommandContext) error {
 
 	r := runtime.NewRuntime(s.Nodes...)
 
-	results := r.Start(tests, ctx.Concurrent)
+	results := r.Start(tests)
 	out := output.NewCliOutput(!ctx.NoColor)
 	if !out.Start(results) {
 		return fmt.Errorf("Test suite failed, use --verbose for more detailed output")

pkg/runtime/docker_executor.go

@@ -18,8 +18,6 @@ const (
 	LineCount = "LineCount"
 )
 
-const WorkerCountMultiplicator = 5
-
 // Result status codes
 const (
 	//Success status
@@ -131,8 +129,7 @@ type TestResult struct {
 }
 
 // Start starts the given test suite and executes all tests
-// maxConcurrent configures the amount of go routines which will be started
-func (r *Runtime) Start(tests []TestCase, maxConcurrent int) <-chan TestResult {
+func (r *Runtime) Start(tests []TestCase) <-chan TestResult {
 	in := make(chan TestCase)
 	out := make(chan TestResult)
 
@@ -213,23 +210,6 @@ func (r *Runtime) getExecutor(node string) Executor {
 	return LocalExecutor{}
 }
 
-func execTest(t TestCase) TestResult {
-	result := TestResult{}
-	for i := 1; i <= t.Command.GetRetries(); i++ {
-
-		e := LocalExecutor{}
-		result = e.Execute(t)
-
-		result.Tries = i
-		if result.ValidationResult.Success {
-			break
-		}
-
-		executeRetryInterval(t)
-	}
-	return result
-}
-
 func executeRetryInterval(t TestCase) {
 	if t.Command.GetRetries() > 1 && t.Command.Interval != "" {
 		interval, err := time.ParseDuration(t.Command.Interval)

pkg/runtime/runtime.go

@@ -6,12 +6,16 @@ import (
 	"time"
 )
 
-const SingleConcurrent = 1
+func Test_NewRuntime(t *testing.T) {
+	runtime := NewRuntime(Node{Name: "test"}, Node{Name: "test2"})
+
+	assert.Len(t, runtime.Nodes, 3)
+}
 
 func TestRuntime_Start(t *testing.T) {
 	s := getExampleTestSuite()
 	r := Runtime{}
-	got := r.Start(s, SingleConcurrent)
+	got := r.Start(s)
 
 	assert.IsType(t, make(<-chan TestResult), got)
 
@@ -30,7 +34,7 @@ func TestRuntime_WithRetries(t *testing.T) {
 	s[0].Command.Cmd = "echo fail"
 
 	r := Runtime{}
-	got := r.Start(s, 1)
+	got := r.Start(s)
 
 	var counter = 0
 	for r := range got {
@@ -50,7 +54,7 @@ func TestRuntime_WithRetriesAndInterval(t *testing.T) {
 
 	start := time.Now()
 	r := Runtime{}
-	got := r.Start(s, 0)
+	got := r.Start(s)
 
 	var counter = 0
 	for r := range got {
@@ -64,6 +68,26 @@ func TestRuntime_WithRetriesAndInterval(t *testing.T) {
 	assert.True(t, duration.Seconds() > 0.15, "Retry interval did not work")
 }
 
+func Test_Runtime_getExecutor(t *testing.T) {
+	r := NewRuntime(
+		Node{Name: "ssh-host", Type: "ssh"},
+		Node{Name: "localhost", Type: "local"},
+		Node{Name: "default", Type: ""},
+	)
+
+	// If empty string set as type use local executor
+	n := r.getExecutor("default")
+	assert.IsType(t, LocalExecutor{}, n)
+
+	n = nil
+	n = r.getExecutor("localhost")
+	assert.IsType(t, LocalExecutor{}, n)
+
+	n = nil
+	n = r.getExecutor("ssh-host")
+	assert.IsType(t, SSHExecutor{}, n)
+}
+
 func getExampleTestSuite() []TestCase {
 	tests := []TestCase{
 		{

pkg/runtime/runtime_test.go

@@ -16,24 +16,14 @@ type SSHExecutorTestEnv struct {
 
 var SSHTestEnv SSHExecutorTestEnv
 
-func TestMain(m *testing.M) {
-	v := os.Getenv("COMMANDER_SSH_TEST")
-	if v != "1" {
-		log.Println("Skip ssh_executor_test, set env COMMANDER_SSH_TEST to 1")
-		return
-	}
-
+func createExecutor() SSHExecutor {
 	SSHTestEnv = SSHExecutorTestEnv{
 		Host:         os.Getenv("COMMANDER_TEST_SSH_HOST"),
 		Pass:         os.Getenv("COMMANDER_TEST_SSH_PASS"),
 		User:         os.Getenv("COMMANDER_TEST_SSH_USER"),
 		IdentityFile: os.Getenv("COMMANDER_TEST_SSH_IDENTITY_FILE"),
 	}
 
-	m.Run()
-}
-
-func createExecutor() SSHExecutor {
 	s := SSHExecutor{
 		Host:         SSHTestEnv.Host,
 		User:         SSHTestEnv.User,
@@ -44,6 +34,10 @@ func createExecutor() SSHExecutor {
 }
 
 func Test_SSHExecutor(t *testing.T) {
+	if !isSSHTestsEnabled() {
+		return
+	}
+
 	s := createExecutor()
 
 	test := TestCase{
@@ -62,6 +56,10 @@ func Test_SSHExecutor(t *testing.T) {
 }
 
 func Test_SSHExecutor_WithDir(t *testing.T) {
+	if !isSSHTestsEnabled() {
+		return
+	}
+
 	s := createExecutor()
 
 	test := TestCase{
@@ -81,6 +79,10 @@ func Test_SSHExecutor_WithDir(t *testing.T) {
 }
 
 func Test_SSHExecutor_ExitCode(t *testing.T) {
+	if !isSSHTestsEnabled() {
+		return
+	}
+
 	s := createExecutor()
 
 	test := TestCase{
@@ -94,3 +96,12 @@ func Test_SSHExecutor_ExitCode(t *testing.T) {
 	assert.False(t, got.ValidationResult.Success)
 	assert.Equal(t, 2, got.TestCase.Result.ExitCode)
 }
+
+func isSSHTestsEnabled() bool {
+	v := os.Getenv("COMMANDER_SSH_TEST")
+	if v != "1" {
+		log.Println("Skip ssh_executor_test, set env COMMANDER_SSH_TEST to 1")
+		return false
+	}
+	return true
+}