PT-2421 - Adding further data collection to pt-k8s-debug-collector

src/go/pt-k8s-debug-collector/README.rst

@@ -117,6 +117,22 @@ Summary, collected for PostgreSQL (available in file summary.txt)
 
    "pg_gather"
 
+Individual files, collected for PostgreSQL (PostgreSQL Operator v2)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   "$PGDATA/log"
+   "pgdata/pgbackrest/log"
+
+Command outputs, collected for PostgreSQL (PostgreSQL Operator v2)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+   "patronictl list"
+   "pgbackrest info"
+
 Usage
 =====
 

src/go/pt-k8s-debug-collector/dumper/dumper.go

@@ -65,12 +65,18 @@ type Dumper struct {
 	restConfig      *rest.Config
 }
 
+type toolLog struct {
+	filename string
+	args     []string
+}
+
 // individualFile struct is used to dump the necessary files from the containers
 type individualFile struct {
 	resourceName  string
 	containerName string
 	filepaths     []string
 	dirpaths      map[string][]string // map[tarFolder][]dirPaths
+	toolCmds      map[string][]toolLog
 }
 
 // resourceMap struct is used to dump the resources from namespace scope or cluster scope
@@ -91,7 +97,7 @@ func New(location, namespace, kubeconfig, clusterName, forwardport, resource str
 	log.AddHook(&ErrorArchiveHook{safeLogger: safeLog})
 
 	if clusterName == "" {
-		_,  clusterName = parseResourceSpec(resource)
+		_, clusterName = parseResourceSpec(resource)
 	}
 
 	config, err := buildRestConfig(kubeconfig, clusterName)

src/go/pt-k8s-debug-collector/dumper/individual_files.go

@@ -68,9 +68,32 @@ func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType s
 				}
 			}
 		}
+
+		for tarFolder, cmds := range indf.toolCmds {
+			for _, cmd := range cmds {
+				if err := d.processToolOutput(ctx, job, indf.containerName, tarFolder, cmd); err != nil {
+					log.Warnf("Skipping tool cmd %v: %v", cmd.args, err)
+				}
+			}
+		}
 	}
 }
 
+func (d *Dumper) processToolOutput(
+	ctx context.Context,
+	job exportJob,
+	container, tarFolder string, cmd toolLog,
+) error {
+	out, stderr, err := d.executeInPod(ctx, cmd.args, job.Pod, container, nil)
+	if err != nil {
+		return fmt.Errorf("exec %v: %w (stderr: %s)", cmd.args, err, stderr.String())
+	}
+
+	dst := d.PodIndividualFilesPath(job.Pod.Namespace, job.Pod.Name, path.Join(tarFolder, cmd.filename))
+
+	return d.archive.WriteVirtualFile(dst, out.Bytes())
+}
+
 func (d *Dumper) processSingleFile(
 	ctx context.Context,
 	job exportJob,

src/go/pt-k8s-debug-collector/dumper/resources.go

@@ -26,13 +26,28 @@ func (d *Dumper) addPg1() error {
 
 func (d *Dumper) addPg2() error {
 	dirpaths := map[string][]string{
-		"pg_log": {"$PGDATA/log"},
+		"pg_log":         {"$PGDATA/log"},
+		"pgbackrest_log": {"pgdata/pgbackrest/log"},
+	}
+
+	tools := map[string][]toolLog{
+		"": {
+			{
+				filename: "patronictl-list.log",
+				args:     []string{"patronictl", "list"},
+			},
+			{
+				filename: "pgbackrest-info.log",
+				args:     []string{"pgbackrest", "info"},
+			},
+		},
 	}
 
 	d.individualFiles = append(d.individualFiles, individualFile{
 		resourceName:  "pgv2",
 		containerName: "database",
 		dirpaths:      dirpaths,
+		toolCmds:      tools,
 	})
 	return nil
 }

src/go/pt-k8s-debug-collector/main_test.go

@@ -300,15 +300,17 @@ func (s *CollectorSuite) TestIndividualFiles() {
 		name         string
 		cmd          []string
 		want         []string
+		skipForNone  bool
 		preprocessor func(string) string
 	}{
 		{
 			namespace: "pxc",
 			// If the tool collects required log files
 			name: "pxc_logs_list",
 			// tar -tf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/*'
-			cmd:  []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/*"},
-			want: []string{"auto.cnf", "grastate.dat", "gvwstate.dat", "innobackup.backup.log", "innobackup.move.log", "innobackup.prepare.log", "mysqld-error.log", "mysqld.post.processing.log"},
+			cmd:         []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/*"},
+			want:        []string{"auto.cnf", "grastate.dat", "gvwstate.dat", "innobackup.backup.log", "innobackup.move.log", "innobackup.prepare.log", "mysqld-error.log", "mysqld.post.processing.log"},
+			skipForNone: true,
 			preprocessor: func(in string) string {
 				files := strings.Split(in, "\n")
 				var result []string
@@ -331,8 +333,9 @@ func (s *CollectorSuite) TestIndividualFiles() {
 			// If MySQL error log is not empty
 			name: "pxc_mysqld_error_log",
 			// tar --to-command="grep -m 1 -o Version:" -xzf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/mysqld-error.log'
-			cmd:  []string{"tar", "--to-command", "grep -m 1 -o Version:", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/mysqld-error.log"},
-			want: []string{"Version:"},
+			cmd:         []string{"tar", "--to-command", "grep -m 1 -o Version:", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/mysqld-error.log"},
+			want:        []string{"Version:"},
+			skipForNone: true,
 			preprocessor: func(in string) string {
 				nl := strings.Index(in, "\n")
 				if nl == -1 {
@@ -346,8 +349,9 @@ func (s *CollectorSuite) TestIndividualFiles() {
 			// If the tool collects PostgreSQL log files
 			name: "pgo_pg_logs_exist",
 			// tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/*/pg_log/*.log'
-			cmd:  []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"},
-			want: []string{".log"},
+			cmd:         []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"},
+			want:        []string{".log"},
+			skipForNone: true,
 			preprocessor: func(in string) string {
 				files := strings.Split(in, "\n")
 				var result []string
@@ -365,8 +369,9 @@ func (s *CollectorSuite) TestIndividualFiles() {
 			// If the tool collects PostgreSQL log files for pgv2
 			name: "pgv2_pg_logs_exist",
 			// tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/*/pg_log/*.log'
-			cmd:  []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"},
-			want: []string{".log"},
+			cmd:         []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"},
+			want:        []string{".log"},
+			skipForNone: true,
 			preprocessor: func(in string) string {
 				files := strings.Split(in, "\n")
 				var result []string
@@ -379,13 +384,70 @@ func (s *CollectorSuite) TestIndividualFiles() {
 				return strings.Join(result, "")
 			},
 		},
+		{
+			namespace:   "pgv2",
+			name:        "pgv2_pgbackrest_log_list",
+			cmd:         []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/pgbackrest_log/*"},
+			want:        []string{"db-archive-push-async.log", "db-stanza-create.log"},
+			skipForNone: true,
+			preprocessor: func(in string) string {
+				required := map[string]struct{}{
+					"db-archive-push-async.log": {},
+					"db-stanza-create.log":      {},
+				}
+
+				files := strings.Split(in, "\n")
+				var result []string
+				for _, f := range files {
+					b := path.Base(f)
+					if _, ok := required[b]; !ok {
+						continue
+					}
+
+					if !slices.Contains(result, b) {
+						result = append(result, b)
+					}
+				}
+				slices.Sort(result)
+				return strings.Join(result, "\n")
+			},
+		},
+		{
+			namespace:   "pgv2",
+			name:        "pgv2_tools_log_list",
+			cmd:         []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/*"},
+			want:        []string{"patronictl-list.log", "pgbackrest-info.log"},
+			skipForNone: true,
+			preprocessor: func(in string) string {
+				required := map[string]struct{}{
+					"patronictl-list.log": {},
+					"pgbackrest-info.log": {},
+				}
+
+				files := strings.Split(in, "\n")
+				var result []string
+				for _, f := range files {
+					b := path.Base(f)
+					if _, ok := required[b]; !ok {
+						continue
+					}
+
+					if !slices.Contains(result, b) {
+						result = append(result, b)
+					}
+				}
+				slices.Sort(result)
+				return strings.Join(result, "\n")
+			},
+		},
 		{
 			namespace: "pxc",
 			// If pod logs are exported as one file per container
 			name: "pxc_container_logs_split_by_container",
 			// tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/pxc/*/*.log'
-			cmd:  []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/pxc/*/*.log"},
-			want: []string{"logrotate.log", "logs.log", "pxc-init.log", "pxc.log"},
+			cmd:         []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/pxc/*/*.log"},
+			want:        []string{"logrotate.log", "logs.log", "pxc-init.log", "pxc.log"},
+			skipForNone: false,
 			preprocessor: func(in string) string {
 				required := map[string]struct{}{
 					"logrotate.log": {},
@@ -424,6 +486,7 @@ func (s *CollectorSuite) TestIndividualFiles() {
 		name         string
 		cmd          []string
 		want         []string
+		skipForNone  bool
 		preprocessor func(string) string
 	}{}
 
@@ -444,10 +507,11 @@ func (s *CollectorSuite) TestIndividualFiles() {
 			s.NoError(err)
 
 			for _, test := range nsTests {
-				out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput()
-				if err != nil && resource == "none" {
+				if resource == "none" && test.skipForNone {
 					continue
 				}
+
+				out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput()
 				s.NoError(err)
 				if test.preprocessor(bytes.NewBuffer(out).String()) != strings.Join(test.want, "\n") {
 					s.Failf("Preprocessor Check", "test %s\nresource:%s\nnamespace: %s\noutput is not as expected\nOutput: %s\nWanted: %s", test.name, resource, test.namespace, test.preprocessor(bytes.NewBuffer(out).String()), test.want)