diff --git a/src/go/pt-k8s-debug-collector/README.rst b/src/go/pt-k8s-debug-collector/README.rst index 95ceaaf69..259563fc1 100644 --- a/src/go/pt-k8s-debug-collector/README.rst +++ b/src/go/pt-k8s-debug-collector/README.rst @@ -117,6 +117,22 @@ Summary, collected for PostgreSQL (available in file summary.txt) "pg_gather" +Individual files, collected for PostgreSQL (PostgreSQL Operator v2) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + "$PGDATA/log" + "pgdata/pgbackrest/log" + +Command outputs, collected for PostgreSQL (PostgreSQL Operator v2) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + "patronictl list" + "pgbackrest info" + Usage ===== diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index 4fe733df6..c5e0f96f5 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -65,12 +65,18 @@ type Dumper struct { restConfig *rest.Config } +type toolLog struct { + filename string + args []string +} + // individualFile struct is used to dump the necessary files from the containers type individualFile struct { resourceName string containerName string filepaths []string dirpaths map[string][]string // map[tarFolder][]dirPaths + toolCmds map[string][]toolLog } // resourceMap struct is used to dump the resources from namespace scope or cluster scope @@ -91,7 +97,7 @@ func New(location, namespace, kubeconfig, clusterName, forwardport, resource str log.AddHook(&ErrorArchiveHook{safeLogger: safeLog}) if clusterName == "" { - _, clusterName = parseResourceSpec(resource) + _, clusterName = parseResourceSpec(resource) } config, err := buildRestConfig(kubeconfig, clusterName) diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index e8bb9916e..2f690035b 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -68,9 +68,32 @@ func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType s } } } + + for tarFolder, cmds := range indf.toolCmds { + for _, cmd := range cmds { + if err := d.processToolOutput(ctx, job, indf.containerName, tarFolder, cmd); err != nil { + log.Warnf("Skipping tool cmd %v: %v", cmd.args, err) + } + } + } } } +func (d *Dumper) processToolOutput( + ctx context.Context, + job exportJob, + container, tarFolder string, cmd toolLog, +) error { + out, stderr, err := d.executeInPod(ctx, cmd.args, job.Pod, container, nil) + if err != nil { + return fmt.Errorf("exec %v: %w (stderr: %s)", cmd.args, err, stderr.String()) + } + + dst := d.PodIndividualFilesPath(job.Pod.Namespace, job.Pod.Name, path.Join(tarFolder, cmd.filename)) + + return d.archive.WriteVirtualFile(dst, out.Bytes()) +} + func (d *Dumper) processSingleFile( ctx context.Context, job exportJob, diff --git a/src/go/pt-k8s-debug-collector/dumper/resources.go b/src/go/pt-k8s-debug-collector/dumper/resources.go index 7fd2f79b6..1dacd60aa 100644 --- a/src/go/pt-k8s-debug-collector/dumper/resources.go +++ b/src/go/pt-k8s-debug-collector/dumper/resources.go @@ -26,13 +26,28 @@ func (d *Dumper) addPg1() error { func (d *Dumper) addPg2() error { dirpaths := map[string][]string{ - "pg_log": {"$PGDATA/log"}, + "pg_log": {"$PGDATA/log"}, + "pgbackrest_log": {"pgdata/pgbackrest/log"}, + } + + tools := map[string][]toolLog{ + "": { + { + filename: "patronictl-list.log", + args: []string{"patronictl", "list"}, + }, + { + filename: "pgbackrest-info.log", + args: []string{"pgbackrest", "info"}, + }, + }, } d.individualFiles = append(d.individualFiles, individualFile{ resourceName: "pgv2", containerName: "database", dirpaths: dirpaths, + toolCmds: tools, }) return nil } diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 4747c4db1..31c8967f1 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -300,6 +300,7 @@ func (s *CollectorSuite) TestIndividualFiles() { name string cmd []string want []string + skipForNone bool preprocessor func(string) string }{ { @@ -307,8 +308,9 @@ func (s *CollectorSuite) TestIndividualFiles() { // If the tool collects required log files name: "pxc_logs_list", // tar -tf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/*' - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/*"}, - want: []string{"auto.cnf", "grastate.dat", "gvwstate.dat", "innobackup.backup.log", "innobackup.move.log", "innobackup.prepare.log", "mysqld-error.log", "mysqld.post.processing.log"}, + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/*"}, + want: []string{"auto.cnf", "grastate.dat", "gvwstate.dat", "innobackup.backup.log", "innobackup.move.log", "innobackup.prepare.log", "mysqld-error.log", "mysqld.post.processing.log"}, + skipForNone: true, preprocessor: func(in string) string { files := strings.Split(in, "\n") var result []string @@ -331,8 +333,9 @@ func (s *CollectorSuite) TestIndividualFiles() { // If MySQL error log is not empty name: "pxc_mysqld_error_log", // tar --to-command="grep -m 1 -o Version:" -xzf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/mysqld-error.log' - cmd: []string{"tar", "--to-command", "grep -m 1 -o Version:", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/mysqld-error.log"}, - want: []string{"Version:"}, + cmd: []string{"tar", "--to-command", "grep -m 1 -o Version:", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/mysqld-error.log"}, + want: []string{"Version:"}, + skipForNone: true, preprocessor: func(in string) string { nl := strings.Index(in, "\n") if nl == -1 { @@ -346,8 +349,9 @@ func (s *CollectorSuite) TestIndividualFiles() { // If the tool collects PostgreSQL log files name: "pgo_pg_logs_exist", // tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/*/pg_log/*.log' - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"}, - want: []string{".log"}, + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"}, + want: []string{".log"}, + skipForNone: true, preprocessor: func(in string) string { files := strings.Split(in, "\n") var result []string @@ -365,8 +369,9 @@ func (s *CollectorSuite) TestIndividualFiles() { // If the tool collects PostgreSQL log files for pgv2 name: "pgv2_pg_logs_exist", // tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/*/pg_log/*.log' - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"}, - want: []string{".log"}, + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"}, + want: []string{".log"}, + skipForNone: true, preprocessor: func(in string) string { files := strings.Split(in, "\n") var result []string @@ -379,13 +384,70 @@ func (s *CollectorSuite) TestIndividualFiles() { return strings.Join(result, "") }, }, + { + namespace: "pgv2", + name: "pgv2_pgbackrest_log_list", + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/pgbackrest_log/*"}, + want: []string{"db-archive-push-async.log", "db-stanza-create.log"}, + skipForNone: true, + preprocessor: func(in string) string { + required := map[string]struct{}{ + "db-archive-push-async.log": {}, + "db-stanza-create.log": {}, + } + + files := strings.Split(in, "\n") + var result []string + for _, f := range files { + b := path.Base(f) + if _, ok := required[b]; !ok { + continue + } + + if !slices.Contains(result, b) { + result = append(result, b) + } + } + slices.Sort(result) + return strings.Join(result, "\n") + }, + }, + { + namespace: "pgv2", + name: "pgv2_tools_log_list", + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/*"}, + want: []string{"patronictl-list.log", "pgbackrest-info.log"}, + skipForNone: true, + preprocessor: func(in string) string { + required := map[string]struct{}{ + "patronictl-list.log": {}, + "pgbackrest-info.log": {}, + } + + files := strings.Split(in, "\n") + var result []string + for _, f := range files { + b := path.Base(f) + if _, ok := required[b]; !ok { + continue + } + + if !slices.Contains(result, b) { + result = append(result, b) + } + } + slices.Sort(result) + return strings.Join(result, "\n") + }, + }, { namespace: "pxc", // If pod logs are exported as one file per container name: "pxc_container_logs_split_by_container", // tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/pxc/*/*.log' - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/pxc/*/*.log"}, - want: []string{"logrotate.log", "logs.log", "pxc-init.log", "pxc.log"}, + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/pxc/*/*.log"}, + want: []string{"logrotate.log", "logs.log", "pxc-init.log", "pxc.log"}, + skipForNone: false, preprocessor: func(in string) string { required := map[string]struct{}{ "logrotate.log": {}, @@ -424,6 +486,7 @@ func (s *CollectorSuite) TestIndividualFiles() { name string cmd []string want []string + skipForNone bool preprocessor func(string) string }{} @@ -444,10 +507,11 @@ func (s *CollectorSuite) TestIndividualFiles() { s.NoError(err) for _, test := range nsTests { - out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput() - if err != nil && resource == "none" { + if resource == "none" && test.skipForNone { continue } + + out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput() s.NoError(err) if test.preprocessor(bytes.NewBuffer(out).String()) != strings.Join(test.want, "\n") { s.Failf("Preprocessor Check", "test %s\nresource:%s\nnamespace: %s\noutput is not as expected\nOutput: %s\nWanted: %s", test.name, resource, test.namespace, test.preprocessor(bytes.NewBuffer(out).String()), test.want)