tests, appveyor: only execute one in four permutations on CI (ethereum#29220)

tests, appveyor: only execute one in four permutations when flag -short is used

Also enable -short flag on all appveyor builds (also ubuntu)

Author: holiman

appveyor.yml

@@ -26,7 +26,7 @@ for:
       - go run build/ci.go lint
       - go run build/ci.go install -dlgo
     test_script:
-      - go run build/ci.go test -dlgo
+      - go run build/ci.go test -dlgo -short
 
   # linux/386 is disabled.
   - matrix:

tests/block_test.go

@@ -18,7 +18,6 @@ package tests
 
 import (
 	"math/rand"
-	"runtime"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -51,9 +50,6 @@ func TestBlockchain(t *testing.T) {
 	bt.skipLoad(`.*randomStatetest94.json.*`)
 
 	bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
-		if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 {
-			t.Skip("test (randomly) skipped on 32-bit windows")
-		}
 		execBlockTest(t, bt, test)
 	})
 	// There is also a LegacyTests folder, containing blockchain tests generated
@@ -74,20 +70,33 @@ func TestExecutionSpecBlocktests(t *testing.T) {
 }
 
 func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) {
-	if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil {
-		t.Errorf("test in hash mode without snapshotter failed: %v", err)
-		return
+	// If -short flag is used, we don't execute all four permutations, only one.
+	executionMask := 0xf
+	if testing.Short() {
+		executionMask = (1 << (rand.Int63() & 4))
+	}
+	if executionMask&0x1 != 0 {
+		if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil {
+			t.Errorf("test in hash mode without snapshotter failed: %v", err)
+			return
+		}
 	}
-	if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil {
-		t.Errorf("test in hash mode with snapshotter failed: %v", err)
-		return
+	if executionMask&0x2 != 0 {
+		if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil {
+			t.Errorf("test in hash mode with snapshotter failed: %v", err)
+			return
+		}
 	}
-	if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil {
-		t.Errorf("test in path mode without snapshotter failed: %v", err)
-		return
+	if executionMask&0x4 != 0 {
+		if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil {
+			t.Errorf("test in path mode without snapshotter failed: %v", err)
+			return
+		}
 	}
-	if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil {
-		t.Errorf("test in path mode with snapshotter failed: %v", err)
-		return
+	if executionMask&0x8 != 0 {
+		if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil {
+			t.Errorf("test in path mode with snapshotter failed: %v", err)
+			return
+		}
 	}
 }

tests/state_test.go

@@ -25,7 +25,6 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -99,15 +98,20 @@ func TestExecutionSpecState(t *testing.T) {
 }
 
 func execStateTest(t *testing.T, st *testMatcher, test *StateTest) {
-	if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 {
-		t.Skip("test (randomly) skipped on 32-bit windows")
-		return
-	}
 	for _, subtest := range test.Subtests() {
 		subtest := subtest
 		key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index)
 
+		// If -short flag is used, we don't execute all four permutations, only
+		// one.
+		executionMask := 0xf
+		if testing.Short() {
+			executionMask = (1 << (rand.Int63() & 4))
+		}
 		t.Run(key+"/hash/trie", func(t *testing.T) {
+			if executionMask&0x1 == 0 {
+				t.Skip("test (randomly) skipped due to short-tag")
+			}
 			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
 				var result error
 				test.Run(subtest, vmconfig, false, rawdb.HashScheme, func(err error, state *StateTestState) {
@@ -117,6 +121,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) {
 			})
 		})
 		t.Run(key+"/hash/snap", func(t *testing.T) {
+			if executionMask&0x2 == 0 {
+				t.Skip("test (randomly) skipped due to short-tag")
+			}
 			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
 				var result error
 				test.Run(subtest, vmconfig, true, rawdb.HashScheme, func(err error, state *StateTestState) {
@@ -132,6 +139,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) {
 			})
 		})
 		t.Run(key+"/path/trie", func(t *testing.T) {
+			if executionMask&0x4 == 0 {
+				t.Skip("test (randomly) skipped due to short-tag")
+			}
 			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
 				var result error
 				test.Run(subtest, vmconfig, false, rawdb.PathScheme, func(err error, state *StateTestState) {
@@ -141,6 +151,9 @@ func execStateTest(t *testing.T, st *testMatcher, test *StateTest) {
 			})
 		})
 		t.Run(key+"/path/snap", func(t *testing.T) {
+			if executionMask&0x8 == 0 {
+				t.Skip("test (randomly) skipped due to short-tag")
+			}
 			withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
 				var result error
 				test.Run(subtest, vmconfig, true, rawdb.PathScheme, func(err error, state *StateTestState) {