New collector based tests (#66)

Create terraform tests that install the collector directly

Author: avilevy18

Dockerfile

@@ -14,15 +14,16 @@
 
 FROM golang:1.23 AS gobuild
 WORKDIR /src
+ARG BUILD_TAGS
 
 # cache deps before copying source so that we don't re-download as much
 COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . .
-RUN CGO_ENABLED=0 go test -tags=e2e -c ./e2etestrunner -o opentelemetry-operations-e2e-testing.test
+RUN set -x; if [ "$BUILD_TAGS" = "e2ecollector" ]; then BUILD_DIRECTORY="e2etestrunner_collector"; else BUILD_TAGS="e2e"; BUILD_DIRECTORY="e2etestrunner"; fi; CGO_ENABLED=0 go test -timeout 3600s -tags=$BUILD_TAGS -c "./$BUILD_DIRECTORY" -o opentelemetry-operations-e2e-testing.test
 
-FROM hashicorp/terraform:light as tfbuild
+FROM hashicorp/terraform:1.13 as tfbuild
 COPY tf /src/tf
 WORKDIR /src/tf
 ENV TF_PLUGIN_CACHE_DIR=/src/tf/terraform-plugin-cache

cloudbuild-collector.yaml

@@ -0,0 +1,35 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds a docker image, tagging it as _DOCKER_TAG (substitution must be
+# provided) and latest.
+
+steps:
+- name: docker
+  env:
+  - DOCKER_BUILDKIT=1
+  args:
+  - build
+  - --tag=${_TEST_RUNNER_IMAGE_NAME}:${_DOCKER_TAG}
+  - --tag=${_TEST_RUNNER_IMAGE_NAME}:latest
+  - --build-arg=BUILDKIT_INLINE_CACHE=1
+  - --build-arg=BUILD_TAGS=e2ecollector
+  - --cache-from=${_TEST_RUNNER_IMAGE_NAME}:latest
+  - .
+
+images: ["${_TEST_RUNNER_IMAGE_NAME}"]
+options:
+  dynamic_substitutions: true
+substitutions:
+  _TEST_RUNNER_IMAGE_NAME: us-central1-docker.pkg.dev/${PROJECT_ID}/e2e-testing/opentelemetry-operations-e2e-testing

cmd/testmatrix/main.go

@@ -25,6 +25,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etesting"
 	"html/template"
 	"log"
 	"os"
@@ -33,7 +34,6 @@ import (
 	"strings"
 
 	"cloud.google.com/go/storage"
-	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etestrunner"
 	"github.com/alexflint/go-arg"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/api/cloudbuild/v1"
@@ -88,7 +88,7 @@ This will fetch recent Cloud Build logs to automatically update the statuses in
 )
 
 type Args struct {
-	e2etestrunner.CmdWithProjectId
+	e2etesting.CmdWithProjectId
 }
 
 type status string
@@ -106,7 +106,7 @@ var (
 )
 
 func main() {
-	args := e2etestrunner.Args{}
+	args := e2etesting.Args{}
 	arg.MustParse(&args)
 
 	ctx := context.Background()

e2etestrunner/art.go e2etesting/art.goe2etestrunner/art.go renamed to e2etesting/art.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package e2etestrunner
+package e2etesting
 
 const BeginOutputArt = `
 

e2etestrunner/e2e_testing.go e2etesting/e2e_testing.goe2etestrunner/e2e_testing.go renamed to e2etesting/e2e_testing.go

@@ -12,14 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package e2etestrunner
+package e2etesting
 
 import (
 	"context"
+	"flag"
 	"log"
+	"math/rand"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etestrunner/testclient"
+	"github.com/alexflint/go-arg"
 )
 
 type ApplyPersistent struct {
@@ -52,6 +57,22 @@ type GceCmd struct {
 	CmdWithImage
 }
 
+type GceCollectorCmd struct {
+	CmdWithImage
+}
+
+type GceCollectorArmCmd struct {
+	CmdWithImage
+}
+
+type GkeCollectorCmd struct {
+	CmdWithImage
+}
+
+type GkeOperatorCollectorCmd struct {
+	CmdWithImage
+}
+
 type GkeCmd struct {
 	CmdWithImage
 }
@@ -72,6 +93,10 @@ type CloudRunCmd struct {
 	CmdWithImage
 }
 
+type CloudRunCollectorCmd struct {
+	CmdWithImage
+}
+
 type CloudFunctionsGen2Cmd struct {
 	// Needed to configure which language will the function instance support
 	Runtime string `arg:"required" help:"Configure the language runtime environment for CloudFunction"`
@@ -87,17 +112,22 @@ type Args struct {
 	// tf/persistent/README.md for details on what is in there.
 	ApplyPersistent *ApplyPersistent `arg:"subcommand:apply-persistent" help:"Terraform apply the resources in tf/persistent and exit (does not run tests)."`
 
-	Local              *LocalCmd              `arg:"subcommand:local" help:"Deploy the test server locally with docker and execute tests"`
-	Gke                *GkeCmd                `arg:"subcommand:gke" help:"Deploy the test server on GKE and execute tests"`
-	Gce                *GceCmd                `arg:"subcommand:gce" help:"Deploy the test server on GCE and execute tests"`
-	Gae                *GaeCmd                `arg:"subcommand:gae" help:"Deploy the test server on GAE and execute tests"`
-	GaeStandard        *GaeStandardCmd        `arg:"subcommand:gae-standard" help:"Deploy the test server on GAE standard and execute tests"`
-	CloudRun           *CloudRunCmd           `arg:"subcommand:cloud-run" help:"Deploy the test server on Cloud Run and execute tests"`
-	CloudFunctionsGen2 *CloudFunctionsGen2Cmd `arg:"subcommand:cloud-functions-gen2" help:"Deploy the test server on Cloud Function (2nd Gen) and execute tests"`
+	Local                *LocalCmd                `arg:"subcommand:local" help:"Deploy the test server locally with docker and execute tests"`
+	Gke                  *GkeCmd                  `arg:"subcommand:gke" help:"Deploy the test server on GKE and execute tests"`
+	Gce                  *GceCmd                  `arg:"subcommand:gce" help:"Deploy the test server on GCE and execute tests"`
+	GceCollector         *GceCollectorCmd         `arg:"subcommand:gce-collector" help:"Deploy the collector on GCE and execute tests"`
+	GceCollectorArm      *GceCollectorArmCmd      `arg:"subcommand:gce-collector-arm" help:"Deploy the collector on GCE and execute tests"`
+	GkeCollector         *GkeCollectorCmd         `arg:"subcommand:gke-collector" help:"Deploy the collector on GKE and execute tests"`
+	GkeOperatorCollector *GkeOperatorCollectorCmd `arg:"subcommand:gke-operator-collector" help:"Deploy the collector on GKE using the OpenTelemetry Operator and execute tests"`
+	Gae                  *GaeCmd                  `arg:"subcommand:gae" help:"Deploy the test server on GAE and execute tests"`
+	GaeStandard          *GaeStandardCmd          `arg:"subcommand:gae-standard" help:"Deploy the test server on GAE standard and execute tests"`
+	CloudRun             *CloudRunCmd             `arg:"subcommand:cloud-run" help:"Deploy the test server on Cloud Run and execute tests"`
+	CloudRunCollector    *CloudRunCollectorCmd    `arg:"subcommand:cloud-run-collector" help:"Deploy the collector on Cloud Run and execute tests"`
+	CloudFunctionsGen2   *CloudFunctionsGen2Cmd   `arg:"subcommand:cloud-functions-gen2" help:"Deploy the test server on Cloud Function (2nd Gen) and execute tests"`
 
 	CmdWithProjectId
 	GoTestFlags        string        `help:"go test flags to pass through, e.g. --gotestflags='-test.v'"`
-	HealthCheckTimeout time.Duration `arg:"--health-check-timeout" help:"A duration (e.g. 5m) to wait for the test server health check. Default is 2m." default:"2m"`
+	HealthCheckTimeout time.Duration `arg:"--health-check-timeout" help:"A duration (e.g. 5m) to wait for the test server health check. Default is 2m." default:"15m"`
 
 	// This is used in a new terraform workspace's name and in the GCP resources
 	// we create. Pass the GCB build ID in CI to get the build id formatted into
@@ -113,4 +143,46 @@ type SetupFunc func(
 	*log.Logger,
 ) (*testclient.Client, Cleanup, error)
 
+type SetupCollectorFunc func(
+	context.Context,
+	*Args,
+	*log.Logger,
+) (Cleanup, error)
+
 func NoopCleanup() {}
+
+type ApplyPersistentFunc func(ctx context.Context, projectID string, autoApprove bool, logger *log.Logger) error
+
+func InitTestMain(args *Args, applyPersistent ApplyPersistentFunc) (*log.Logger, context.Context) {
+	rand.New(rand.NewSource(time.Now().UnixNano()))
+	p := arg.MustParse(args)
+	if p.Subcommand() == nil {
+		p.Fail("missing command")
+	}
+	// Need a logger just for TestMain() before testing.T is available
+	logger := log.New(os.Stdout, "TestMain: ", log.LstdFlags|log.Lshortfile)
+	ctx := context.Background()
+
+	// Handle special case of just creating persistent resources
+	if args.ApplyPersistent != nil {
+		err := applyPersistent(ctx, args.ProjectID, args.ApplyPersistent.AutoApprove, logger)
+		if err != nil {
+			logger.Panic(err)
+		}
+		return nil, nil
+	}
+
+	// hacky but works
+	os.Args = append([]string{os.Args[0]}, strings.Fields(args.GoTestFlags)...)
+	flag.Parse()
+
+	// handle any complex defaults
+	if args.TestRunID == "" {
+		hex, err := RandomHex(6)
+		if err != nil {
+			logger.Fatalf("error generating random hex string: %v\n", err)
+		}
+		args.TestRunID = hex
+	}
+	return logger, ctx
+}

e2etestrunner/setuptf/setuptf.go e2etesting/setuptf/setuptf.goe2etestrunner/setuptf/setuptf.go renamed to e2etesting/setuptf/setuptf.go

@@ -24,9 +24,10 @@ import (
 )
 
 const (
-	tfPersistentDir                  = "tf/persistent"
-	Push            SubscriptionMode = "push"
-	Pull            SubscriptionMode = "pull"
+	tfPersistentDir                           = "tf/persistent"
+	tfPersistentCollectorDir                  = "tf/persistent-collector"
+	Push                     SubscriptionMode = "push"
+	Pull                     SubscriptionMode = "pull"
 )
 
 type SubscriptionMode string
@@ -94,6 +95,8 @@ func SetupTf(
 		return nil, func() {}, err
 	}
 
+	logger.Printf("Running %s with image: %s\n", tfDir, tfVars["image"])
+
 	cleanup := func() {
 		defer deleteWorkspace(ctx, testRunID, tfDir, logger)
 
@@ -155,18 +158,37 @@ func SetupTf(
 	return &tfOutput.PubsubInfoWrapper.Value, cleanup, nil
 }
 
+func ApplyPersistent(
+	ctx context.Context,
+	projectID string,
+	autoApprove bool,
+	logger *log.Logger,
+) error {
+	return applyPersistent(ctx, projectID, autoApprove, logger, tfPersistentDir)
+}
+
+func ApplyPersistentCollector(
+	ctx context.Context,
+	projectID string,
+	autoApprove bool,
+	logger *log.Logger,
+) error {
+	return applyPersistent(ctx, projectID, autoApprove, logger, tfPersistentCollectorDir)
+}
+
 // Create persistent resources (in tf/persistent) that are used across tests. No
 // cleanup is required
-func ApplyPersistent(
+func applyPersistent(
 	ctx context.Context,
 	projectID string,
 	autoApprove bool,
 	logger *log.Logger,
+	persistentDir string,
 ) error {
 	logger.Println("Applying any changes to persistent resources")
 	// Run terraform init
 	cmd := initCommand(ctx, projectID)
-	cmd.Dir = tfPersistentDir
+	cmd.Dir = persistentDir
 	if err := runWithOutput(cmd, logger); err != nil {
 		return err
 	}

e2etestrunner/util.go e2etesting/util.goe2etestrunner/util.go renamed to e2etesting/util.go

@@ -12,15 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package e2etestrunner
+package e2etesting
 
 import (
+	"crypto/rand"
 	"encoding/hex"
-	"math/rand"
 )
 
 // Generates a random hex string of given length
-func randomHex(length uint8) (string, error) {
+func RandomHex(length uint8) (string, error) {
 	bytes := make([]byte, length)
 	if _, err := rand.Read(bytes); err != nil {
 		return "", err

e2etestrunner/main_test.go

@@ -19,57 +19,23 @@ package e2etestrunner
 
 import (
 	"context"
-	"flag"
-	"log"
-	"math/rand"
-	"os"
-	"strings"
 	"testing"
-	"time"
 
-	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etestrunner/setuptf"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etesting"
+	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etesting/setuptf"
+
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-e2e-testing/e2etestrunner/testclient"
-	"github.com/alexflint/go-arg"
 )
 
 var (
-	args             Args
+	args             e2etesting.Args
 	testServerClient *testclient.Client
 )
 
 func TestMain(m *testing.M) {
-	rand.Seed(time.Now().UnixNano())
-	p := arg.MustParse(&args)
-	if p.Subcommand() == nil {
-		p.Fail("missing command")
-	}
-	// Need a logger just for TestMain() before testing.T is available
-	logger := log.New(os.Stdout, "TestMain: ", log.LstdFlags|log.Lshortfile)
-	ctx := context.Background()
-
-	// Handle special case of just creating persistent resources
-	if args.ApplyPersistent != nil {
-		err := setuptf.ApplyPersistent(ctx, args.ProjectID, args.ApplyPersistent.AutoApprove, logger)
-		if err != nil {
-			logger.Panic(err)
-		}
-		return
-	}
-
-	// hacky but works
-	os.Args = append([]string{os.Args[0]}, strings.Fields(args.GoTestFlags)...)
-	flag.Parse()
-
-	// handle any complex defaults
-	if args.TestRunID == "" {
-		hex, err := randomHex(6)
-		if err != nil {
-			logger.Fatalf("error generating random hex string: %v\n", err)
-		}
-		args.TestRunID = hex
-	}
+	logger, ctx := e2etesting.InitTestMain(&args, setuptf.ApplyPersistent)
 
-	var setupFunc SetupFunc
+	var setupFunc e2etesting.SetupFunc
 	switch {
 	case args.Local != nil:
 		setupFunc = SetupLocal
@@ -106,7 +72,7 @@ func TestMain(m *testing.M) {
 	}
 
 	// Run tests
-	logger.Print(BeginOutputArt)
+	logger.Print(e2etesting.BeginOutputArt)
 	m.Run()
-	logger.Print(EndOutputArt)
+	logger.Print(e2etesting.EndOutputArt)
 }