Support save pod/container std log on the Environment (#62)

Author: mrproliu

action.yaml

@@ -21,6 +21,9 @@ inputs:
   e2e-file:
     description: File path of e2e file
     required: true
+  log-dir:
+    description: The container logs directory
+    required: false
 
 runs:
   using: "composite"
@@ -30,5 +33,46 @@ runs:
         make -C $GITHUB_ACTION_PATH docker
         docker run -v $(pwd):/tmp -w /tmp --entrypoint=sh docker.io/apache/e2e:latest -c "cp /usr/local/bin/e2e /tmp/e2e"
         install ./e2e /usr/local/bin
+    - name: E2E Dir Generator
+      id: 'e2e-dir-generator'
+      shell: bash
+      run: |
+        WORK_DIR="${{ runner.temp }}/skywalking-infra-e2e"
+        echo "::set-output name=work::$WORK_DIR"
+
+        LOG_DIR=""
+        LOG_JOB_DIR=""
+        if [[ "${{ inputs.log-dir }}" == "" ]]
+        then
+          matrix='${{ toJSON(matrix) }}'
+          if [[ "$matrix" == "null" ]]
+          then
+            LOG_DIR="$WORK_DIR/logs"
+            LOG_JOB_DIR="$LOG_DIR/${{ github.job }}"
+          else
+            combine_matrix=$(echo $matrix|jq -r 'to_entries|map(.value)|tostring')
+            # remove json syntax
+            combine_matrix=`echo $combine_matrix|sed -e 's/\[\|\]\|\"//g'`
+            combine_matrix=`echo $combine_matrix|sed -e 's/[\{|\}]//g'`
+            # replace to path
+            combine_matrix=`echo $combine_matrix|sed -e 's/[^A-Za-z0-9_-]/_/g'`
+            LOG_DIR="$WORK_DIR/logs"
+            LOG_JOB_DIR="$LOG_DIR/${{ github.job }}_$combine_matrix"
+          fi
+        elif [[ "${{ inputs.log-dir }}" == /* ]]
+        then
+          LOG_DIR="${{ inputs.log-dir }}"
+          LOG_JOB_DIR="${{ inputs.log-dir }}"
+        else
+          LOG_DIR="$WORK_DIR/${{ inputs.log-dir }}"
+          LOG_JOB_DIR="$WORK_DIR/${{ inputs.log-dir }}"
+        fi
+        echo "::set-output name=log::$LOG_DIR"
+        echo "::set-output name=log-case::$LOG_JOB_DIR"
+        echo "SW_INFRA_E2E_LOG_DIR=$LOG_DIR" >> $GITHUB_ENV
     - shell: bash
-      run: e2e run -c "${{ inputs.e2e-file }}"
+      run: |
+        e2e run \
+          -c "${{ inputs.e2e-file }}" \
+          -w "${{ steps.e2e-dir-generator.outputs.work }}" \
+          -l "${{ steps.e2e-dir-generator.outputs.log-case }}"

commands/root.go

@@ -55,18 +55,32 @@ var Root = &cobra.Command{
 		}
 		logger.Log.SetLevel(level)
 
-		util.WorkDir = util.ExpandFilePath(util.WorkDir)
-		if _, err := os.Stat(util.WorkDir); os.IsNotExist(err) {
-			if err := os.MkdirAll(util.WorkDir, os.ModePerm); err != nil {
-				logger.Log.Warnf("failed to create working directory %v", util.WorkDir)
-				return err
-			}
+		util.WorkDir, err = ExpandPathAndCreate(util.WorkDir)
+		if err != nil {
+			logger.Log.Warnf("failed to create working directory %v", util.WorkDir)
+			return err
+		}
+
+		util.LogDir, err = ExpandPathAndCreate(util.LogDir)
+		if err != nil {
+			logger.Log.Warnf("failed to create logging directory %v", util.LogDir)
+			return err
 		}
 
 		return nil
 	},
 }
 
+func ExpandPathAndCreate(path string) (string, error) {
+	path = util.ExpandFilePath(path)
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		if err := os.MkdirAll(path, os.ModePerm); err != nil {
+			return path, err
+		}
+	}
+	return path, nil
+}
+
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() error {
@@ -78,6 +92,7 @@ func Execute() error {
 
 	Root.PersistentFlags().StringVarP(&verbosity, "verbosity", "v", logrus.InfoLevel.String(), "log level (debug, info, warn, error, fatal, panic")
 	Root.PersistentFlags().StringVarP(&util.WorkDir, "work-dir", "w", "~/.skywalking-infra-e2e", "the working directory for skywalking-infra-e2e")
+	Root.PersistentFlags().StringVarP(&util.LogDir, "log-dir", "l", "~/.skywalking-infra-e2e/logs", "the container logs directory for environment")
 	Root.PersistentFlags().StringVarP(&util.CfgFile, "config", "c", constant.E2EDefaultFile, "the config file")
 
 	return Root.Execute()

commands/setup/setup.go

@@ -38,6 +38,7 @@ var Setup = &cobra.Command{
 			return err
 		}
 
+		defer setup.CloseLogFollower()
 		if err := DoSetupAccordingE2E(); err != nil {
 			return fmt.Errorf("[Setup] %s", err)
 		}
@@ -61,6 +62,7 @@ func DoSetupAccordingE2E() error {
 
 	e2eConfig := config.GlobalConfig.E2EConfig
 
+	setup.InitLogFollower()
 	if e2eConfig.Setup.Env == constant.Kind {
 		err := setup.KindSetup(&e2eConfig)
 		if err != nil {
@@ -80,6 +82,8 @@ func DoSetupAccordingE2E() error {
 }
 
 func DoStopSetup() {
+	// close log follower
+	setup.CloseLogFollower()
 	// notify clean up
 	setup.KindCleanNotify()
 }

docs/en/setup/Configuration-File.md

@@ -85,6 +85,10 @@ If you want to access the resource from host, should follow these steps:
       url: http://${pod_foo_host}:${pod_foo_8080}/
    ```
 
+#### Log
+
+The console output of each pod could be found in `${workDir}/logs/${namespace}/${podName}.log`.
+
 ### Compose
 
 ```yaml
@@ -123,6 +127,10 @@ If you want to get the service host and port mapping, should follow these steps:
       url: http://${oap_host}:${oap_8080}/
    ```
 
+#### Log
+
+The console output of each service could be found in `${workDir}/logs/{serviceName}/std.log`.
+
 ## Trigger
 
 After the `Setup` step is finished, use the `Trigger` step to generate traffic.

docs/en/setup/Run-E2E-Tests.md

@@ -31,9 +31,23 @@ e2e cleanup
 
 To use skywalking-infra-e2e in GitHub Actions, add a step in your GitHub workflow.
 
+The working directory could be uploaded to GitHub Action Artifact after the task is completed, which contains environment variables and container logs in the environment.
+
 ```yaml
 - name: Run E2E Test
   uses: apache/skywalking-infra-e2e@main      # always prefer to use a revision instead of `main`.
   with:
-    e2e-file: e2e.yaml                        # need to run E2E file path
+    e2e-file: e2e.yaml                        # (required)need to run E2E file path
+    log-dir: /path/to/log/dir                 # (Optional)Use `<work_dir>/logs/<job_name>_<matrix_value>`(if have GHA matrix) or `<work_dir>/logs/<job_name>` in GHA, and output logs into `<work_dir>/logs` out of GHA env, such as running locally.
+```
+
+If you want to upload the log directory to the GitHub Action Artifact when this E2E test failure, you could define the below content in your GitHub Action Job.
+
+```yaml
+- name: Upload E2E Log
+  uses: actions/upload-artifact@v2
+  if: ${{ failure() }}                      # Only upload the artifact when E2E testing failure
+  with:
+    name: e2e-log
+    path: "${{ env.SW_INFRA_E2E_LOG_DIR }}" # The SkyWalking Infra E2E action sets SW_INFRA_E2E_LOG_DIR automatically. 
 ```

internal/components/setup/common.go

@@ -19,20 +19,21 @@
 package setup
 
 import (
+	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 	"time"
 
-	"k8s.io/client-go/dynamic"
-	"k8s.io/client-go/kubernetes"
-
 	"github.com/apache/skywalking-infra-e2e/internal/config"
 	"github.com/apache/skywalking-infra-e2e/internal/logger"
 	"github.com/apache/skywalking-infra-e2e/internal/util"
 )
 
+var (
+	logFollower *util.ResourceLogFollower
+)
+
 func RunStepsAndWait(steps []config.Step, waitTimeout time.Duration, k8sCluster *util.K8sClusterInfo) error {
 	logger.Log.Debugf("wait timeout is %v", waitTimeout.String())
 
@@ -50,7 +51,7 @@ func RunStepsAndWait(steps []config.Step, waitTimeout time.Duration, k8sCluster
 				Path:  step.Path,
 				Waits: step.Waits,
 			}
-			err := createManifestAndWait(k8sCluster.Client, k8sCluster.Interface, manifest, waitTimeout)
+			err := createManifestAndWait(k8sCluster, manifest, waitTimeout)
 			if err != nil {
 				return err
 			}
@@ -60,7 +61,7 @@ func RunStepsAndWait(steps []config.Step, waitTimeout time.Duration, k8sCluster
 				Waits:   step.Waits,
 			}
 
-			err := RunCommandsAndWait(command, waitTimeout)
+			err := RunCommandsAndWait(command, waitTimeout, k8sCluster)
 			if err != nil {
 				return err
 			}
@@ -79,16 +80,11 @@ func RunStepsAndWait(steps []config.Step, waitTimeout time.Duration, k8sCluster
 }
 
 // createManifestAndWait creates manifests in k8s cluster and concurrent waits according to the manifests' wait conditions.
-func createManifestAndWait(c *kubernetes.Clientset, dc dynamic.Interface, manifest config.Manifest, timeout time.Duration) error {
+func createManifestAndWait(c *util.K8sClusterInfo, manifest config.Manifest, timeout time.Duration) error {
 	waitSet := util.NewWaitSet(timeout)
 
-	kubeConfigYaml, err := ioutil.ReadFile(kubeConfigPath)
-	if err != nil {
-		return err
-	}
-
 	waits := manifest.Waits
-	err = createByManifest(c, dc, manifest)
+	err := createByManifest(c, manifest)
 	if err != nil {
 		return err
 	}
@@ -103,7 +99,7 @@ func createManifestAndWait(c *kubernetes.Clientset, dc dynamic.Interface, manife
 		wait := waits[idx]
 		logger.Log.Infof("waiting for %+v", wait)
 
-		options, err := getWaitOptions(kubeConfigYaml, &wait)
+		options, err := getWaitOptions(c, &wait)
 		if err != nil {
 			return err
 		}
@@ -131,7 +127,7 @@ func createManifestAndWait(c *kubernetes.Clientset, dc dynamic.Interface, manife
 }
 
 // RunCommandsAndWait Concurrently run commands and wait for conditions.
-func RunCommandsAndWait(run config.Run, timeout time.Duration) error {
+func RunCommandsAndWait(run config.Run, timeout time.Duration, cluster *util.K8sClusterInfo) error {
 	waitSet := util.NewWaitSet(timeout)
 
 	commands := run.Command
@@ -140,7 +136,7 @@ func RunCommandsAndWait(run config.Run, timeout time.Duration) error {
 	}
 
 	waitSet.WaitGroup.Add(1)
-	go executeCommandsAndWait(commands, run.Waits, waitSet)
+	go executeCommandsAndWait(commands, run.Waits, waitSet, cluster)
 
 	go func() {
 		waitSet.WaitGroup.Wait()
@@ -160,7 +156,7 @@ func RunCommandsAndWait(run config.Run, timeout time.Duration) error {
 	return nil
 }
 
-func executeCommandsAndWait(commands string, waits []config.Wait, waitSet *util.WaitSet) {
+func executeCommandsAndWait(commands string, waits []config.Wait, waitSet *util.WaitSet, cluster *util.K8sClusterInfo) {
 	defer waitSet.WaitGroup.Done()
 
 	// executes commands
@@ -177,13 +173,7 @@ func executeCommandsAndWait(commands string, waits []config.Wait, waitSet *util.
 		wait := waits[idx]
 		logger.Log.Infof("waiting for %+v", wait)
 
-		kubeConfigYaml, err := ioutil.ReadFile(kubeConfigPath)
-		if err != nil {
-			err = fmt.Errorf("read kube config failed: %s", err)
-			waitSet.ErrChan <- err
-		}
-
-		options, err := getWaitOptions(kubeConfigYaml, &wait)
+		options, err := getWaitOptions(cluster, &wait)
 		if err != nil {
 			err = fmt.Errorf("commands: [%s] get wait options error: %s", commands, err)
 			waitSet.ErrChan <- err
@@ -213,3 +203,13 @@ func GetIdentity() string {
 	}
 	return runID
 }
+
+func InitLogFollower() {
+	logFollower = util.NewResourceLogFollower(context.Background(), util.LogDir)
+}
+
+func CloseLogFollower() {
+	if logFollower != nil {
+		logFollower.Close()
+	}
+}