From f8c58ad27023cc2b4a09df0ba47df69566de333d Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Tue, 6 Jan 2026 16:15:11 +0200 Subject: [PATCH 01/21] PT-2237 - fixed PostgreSQL DB logs collection This fix ensures `pt-k8s-debug-collector` collects postgreSql databse logs which is stored in `/pgdata//pg_log`. A test `TestIndividualFiles` was refactored, and modified for a new case with new feature. --- .../pt-k8s-debug-collector/dumper/dumper.go | 132 +++++++++++++-- src/go/pt-k8s-debug-collector/main_test.go | 155 ++++++++++++++---- 2 files changed, 240 insertions(+), 47 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index a5e2958e3..bc1642942 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -6,6 +6,8 @@ import ( "compress/gzip" "encoding/base64" "encoding/json" + "fmt" + "io" "log" "os" "os/exec" @@ -32,6 +34,7 @@ type Dumper struct { kubeconfig string resources []string filePaths []string + dirPath []string fileContainer string namespace string location string @@ -126,8 +129,13 @@ func New(location, namespace, resource string, kubeconfig string, forwardport st } sslSecrets := make([]sslSecret, 0) filePaths := make([]string, 0) + dirPaths := make([]string, 0) switch resourceType(resource) { case "pg": + dirPaths = append(dirPaths, + "$PGBACKREST_DB_PATH/pg_log", + ) + d.fileContainer = "database" sslSecrets = append(sslSecrets, sslSecret{ secret: "{{ .Name }}-ssl-ca", @@ -228,6 +236,7 @@ func New(location, namespace, resource string, kubeconfig string, forwardport st d.sslSecrets = sslSecrets d.crType = resource d.filePaths = filePaths + d.dirPath = dirPaths return d } @@ -342,6 +351,7 @@ func (d *Dumper) DumpCluster() error { log.Printf("Error: get %s resource: %v", resource, err) } } + if pod.Labels["app.kubernetes.io/component"] == component || (component == "pg" && pod.Labels["pgo-pg-database"] == "true") || (component == "pgv2" && pod.Labels["pgv2.percona.com/version"] != "" && pod.Labels["postgres-operator.crunchydata.com/instance"] != "") { @@ -380,6 +390,14 @@ func (d *Dumper) DumpCluster() error { log.Printf("Error: get %s file: %v", path, err) } } + + for _, path := range d.dirPath { + err = d.getAllFilesFormDirectory(ns.Name, pod.Name, path, location, tw) + if err != nil { + d.logError(err.Error(), "get file "+path+" for pod "+pod.Name) + log.Printf("Error: get %s file: %v", path, err) + } + } } } @@ -408,9 +426,11 @@ func (d *Dumper) DumpCluster() error { // runCmd run command (Dumper.cmd) with given args, return it output func (d *Dumper) runCmd(args ...string) ([]byte, error) { + baseArgs := []string{"--kubeconfig", d.kubeconfig} + baseArgs = append(baseArgs, args...) + var outb, errb bytes.Buffer - args = append(args, "--kubeconfig", d.kubeconfig) - cmd := exec.Command(d.cmd, args...) + cmd := exec.Command(d.cmd, baseArgs...) cmd.Stdout = &outb cmd.Stderr = &errb err := cmd.Run() @@ -483,6 +503,7 @@ func (d *Dumper) getIndividualFiles(namespace string, podName, path, location st if len(d.fileContainer) == 0 { return errors.Errorf("Logs container name is not specified for resource %s in namespace %s", resourceType(d.crType), d.namespace) } + args := []string{"-n", namespace, "-c", d.fileContainer, "cp", podName + ":" + path, "/dev/stdout"} output, err := d.runCmd(args...) if err != nil { @@ -497,6 +518,95 @@ func (d *Dumper) getIndividualFiles(namespace string, podName, path, location st return addToArchive(location+"/"+path, d.mode, output, tw) } +func (d *Dumper) parseEnvs(namespace, podName, env string) (string, error) { + args := []string{ + "-n", namespace, "-c", d.fileContainer, "exec", podName, "--", + "sh", "-c", fmt.Sprintf("echo %s", env), + } + + fmt.Printf("DBG: running parseEnvs: %v\n", args) + out, err := d.runCmd(args...) + if err != nil { + return "", err + } + fmt.Printf("DBG: running parseEnvs output: %v\n", string(out)) + + resolved := strings.TrimSpace(string(out)) + return resolved, nil +} + +func (d *Dumper) getAllFilesFormDirectory(namespace string, podName, path, location string, tw *tar.Writer) error { + if len(d.fileContainer) == 0 { + return errors.Errorf("Logs container name is not specified for resource %s in namespace %s", resourceType(d.crType), d.namespace) + } + + var err error + path, err = d.parseEnvs(namespace, podName, path) + if err != nil { + log.Printf("Error: parse envs in %s for resource %s in namespace %s: %v", path, resourceType(d.crType), d.namespace, err) + return addToArchive(location, d.mode, []byte(err.Error()), tw) + } + + if len(path) == 0 { + return nil + } + + args := []string{ + "-n", namespace, "-c", d.fileContainer, "exec", podName, "--", + "sh", "-c", fmt.Sprintf("tar cf - -C %s .", path), + } + + out, err := d.runCmd(args...) + if err != nil { + d.logError(err.Error(), args...) + log.Printf("Error: get path %s for resource %s in namespace %s: %v", path, resourceType(d.crType), d.namespace, err) + return addToArchive(location, d.mode, []byte(err.Error()), tw) + } + + if len(out) == 0 { + return nil + } + + tr := tar.NewReader(bytes.NewBuffer(out)) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + d.logError(err.Error(), args...) + log.Printf("Error: get path %s for resource %s in namespace %s: %v", path, resourceType(d.crType), d.namespace, err) + return addToArchive(location, d.mode, []byte(err.Error()), tw) + } + + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeSymlink { + continue + } + + newHdr := &tar.Header{ + Name: filepath.Join(location, path, hdr.Name), + Mode: hdr.Mode, + Size: hdr.Size, + ModTime: hdr.ModTime, + } + + if err := tw.WriteHeader(newHdr); err != nil { + d.logError(err.Error(), args...) + log.Printf("Error: get path %s for resource %s in namespace %s: %v", path, resourceType(d.crType), d.namespace, err) + return addToArchive(location, d.mode, []byte(err.Error()), tw) + } + + if _, err := io.Copy(tw, tr); err != nil { + d.logError(err.Error(), args...) + log.Printf("Error: get path %s for resource %s in namespace %s: %v", path, resourceType(d.crType), d.namespace, err) + return addToArchive(location, d.mode, []byte(err.Error()), tw) + } + } + + return nil +} + func (d *Dumper) getPodSummary(resource, podName, crName string, namespace string) ([]byte, error) { var ( summCmdName string @@ -561,13 +671,13 @@ func (d *Dumper) getPodSummary(resource, podName, crName string, namespace strin summCmdArgs = []string{"--username=" + user, "--password=" + string(pass), "--authenticationDatabase=admin", "127.0.0.1:" + port} } - cmdPortFwd := exec.Command(d.cmd, "port-forward", "pod/"+podName, ports, "-n", namespace, "--kubeconfig", d.kubeconfig) - go func() { - err := cmdPortFwd.Run() - if err != nil { - d.logError(err.Error(), "port-forward") - } - }() + argPortFwd := []string{"port-forward", "pod/" + podName, ports, "-n", namespace, "--kubeconfig", d.kubeconfig} + cmdPortFwd := exec.Command(d.cmd, argPortFwd...) + err := cmdPortFwd.Start() + if err != nil { + return nil, err + } + defer func() { err := cmdPortFwd.Process.Kill() if err != nil { @@ -575,13 +685,13 @@ func (d *Dumper) getPodSummary(resource, podName, crName string, namespace strin } }() - time.Sleep(3 * time.Second) // wait for port-forward command + time.Sleep(10 * time.Second) // wait for port-forward command var outb, errb bytes.Buffer cmd := exec.Command(summCmdName, summCmdArgs...) cmd.Stdout = &outb cmd.Stderr = &errb - err := cmd.Run() + err = cmd.Run() if err != nil { return nil, errors.Wrapf(err, "stderr: %s\nstdout: %s", errb.String(), outb.String()) } diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index a64e5fddf..01deb746c 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path" "regexp" + "sort" "strings" "testing" @@ -41,75 +42,157 @@ Tests TODO: function or create a mock cluster, or find a better way to deploy test clusters. */ +type Resource struct { + name string + env string +} + +var ( + PxcResource = Resource{name: "pxc", env: "KUBECONFIG_PXC"} + PgResource = Resource{name: "pg", env: "KUBECONFIG_PG"} + AutoResource = Resource{name: "auto", env: "KUBECONFIG_PXC"} +) + +type Matcher interface { + Match(t *testing.T, got string) +} + +type ExactMatch struct { + Want []string +} + +func (m ExactMatch) Match(t *testing.T, got string) { + want := strings.Join(m.Want, "\n") + if got != want { + t.Fatalf("output mismatch\nGot:\n%s\nWant:\n%s", got, want) + } +} + +type RegexMatch struct { + Pattern *regexp.Regexp +} + +func (m RegexMatch) Match(t *testing.T, got string) { + for line := range strings.SplitSeq(got, "\n") { + if m.Pattern.MatchString(line) { + return + } + } + t.Fatalf("no line matches pattern %s\nGot:\n%s", m.Pattern, got) +} + +func uniqueBasenames(in string) string { + files := strings.Split(in, "\n") + var result []string + for _, f := range files { + b := path.Base(f) + if !slices.Contains(result, b) && b != "." && b != "" { + result = append(result, b) + } + } + sort.Strings(result) + return strings.Join(result, "\n") +} + +func firstLine(in string) string { + nl := strings.Index(in, "\n") + if nl == -1 { + return in + } + return in[:nl] +} + /* -Tests collection of the individual files by pt-k8s-debug-collector. -Requires running K8SPXC instance and kubectl, configured to access that instance by default. + Tests collection of the individual files by pt-k8s-debug-collector. + Requires running K8SPXC instance and kubectl, configured to access that instance by default. + If some of the env (KUBECONFIG_PXC, KUBECONFIG_PG) is not defined, theese tests will be skiped. */ + func TestIndividualFiles(t *testing.T) { - if os.Getenv("KUBECONFIG_PXC") == "" { - t.Skip("TestIndividualFiles requires K8SPXC") - } tests := []struct { name string cmd []string - want []string + env string preprocessor func(string) string + match Matcher }{ { - // If the tool collects required log files + // If the tool collects required mysql log files name: "pxc_logs_list", // tar -tf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/*' - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/*"}, - want: []string{"auto.cnf", "grastate.dat", "gvwstate.dat", "innobackup.backup.log", "innobackup.move.log", "innobackup.prepare.log", "mysqld-error.log", "mysqld.post.processing.log"}, - preprocessor: func(in string) string { - files := strings.Split(in, "\n") - var result []string - for _, f := range files { - b := path.Base(f) - if !slices.Contains(result, b) && b != "." && b != "" { - result = append(result, b) - } - } - slices.Sort(result) - return strings.Join(result, "\n") + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/*"}, + env: "KUBECONFIG_PXC", + preprocessor: uniqueBasenames, + match: ExactMatch{ + Want: []string{ + "auto.cnf", + "grastate.dat", + "gvwstate.dat", + "innobackup.backup.log", + "innobackup.move.log", + "innobackup.prepare.log", + "mysqld-error.log", + "mysqld.post.processing.log", + }, }, }, { // If MySQL error log is not empty name: "pxc_mysqld_error_log", // tar --to-command="grep -m 1 -o Version:" -xzf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/mysqld-error.log' - cmd: []string{"tar", "--to-command", "grep -m 1 -o Version:", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/mysqld-error.log"}, - want: []string{"Version:"}, - preprocessor: func(in string) string { - nl := strings.Index(in, "\n") - if nl == -1 { - return "" - } - return in[:nl] + cmd: []string{"tar", "--to-command", "grep -m 1 -o Version:", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/var/lib/mysql/mysqld-error.log"}, + env: "KUBECONFIG_PXC", + preprocessor: firstLine, + match: ExactMatch{ + Want: []string{"Version:"}, + }, + }, + { + // if the tool collects required pg log files + name: "pg_logs_list", + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/pgdata/*"}, + env: "KUBECONFIG_PG", + preprocessor: uniqueBasenames, + match: RegexMatch{ + Pattern: regexp.MustCompile(`^postgresql-[A-Za-z]{3}\.log$`), }, }, } - for _, resource := range []string{"pxc", "auto"} { - cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", os.Getenv("KUBECONFIG_PXC"), "--forwardport", os.Getenv("FORWARDPORT"), "--resource", resource) + for _, resource := range []Resource{PxcResource, PgResource, AutoResource} { + if os.Getenv(resource.env) == "" { + t.Logf("TestIndividualFiles requires %s env", resource.env) + continue + } + + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", + "--kubeconfig", os.Getenv(resource.env), + "--forwardport", os.Getenv("FORWARDPORT"), + "--resource", resource.name, + ) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } + defer func() { - cmd = exec.Command("rm", "-f", "cluster-dump.tar.gz") - if err := cmd.Run(); err != nil { + clean := exec.Command("rm", "-f", "cluster-dump.tar.gz") + if err := clean.Run(); err != nil { t.Errorf("error cleaning up test data: %s", err.Error()) } }() for _, test := range tests { + if resource.env != test.env { + continue + } + out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput() if err != nil { - t.Errorf("test %s, error running command %s:\n%s\n\nCommand output:\n%s", test.name, test.cmd[0], err.Error(), out) - } - if test.preprocessor(bytes.NewBuffer(out).String()) != strings.Join(test.want, "\n") { - t.Errorf("test %s, output is not as expected\nOutput: %s\nWanted: %s", test.name, test.preprocessor(bytes.NewBuffer(out).String()), test.want) + t.Errorf("test %s, error running command %s:\n%s\nOutput:\n%s", test.name, test.cmd[0], err.Error(), out) } + + res := test.preprocessor(string(out)) + test.match.Match(t, res) } } } From 2a99d7b7dd630ecd9bc1b2f636957b8bb0d55818 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Tue, 6 Jan 2026 18:53:21 +0200 Subject: [PATCH 02/21] fixes --- src/go/pt-k8s-debug-collector/dumper/dumper.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index bc1642942..0bf82886b 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -392,7 +392,7 @@ func (d *Dumper) DumpCluster() error { } for _, path := range d.dirPath { - err = d.getAllFilesFormDirectory(ns.Name, pod.Name, path, location, tw) + err = d.getAllFilesFromDirectory(ns.Name, pod.Name, path, location, tw) if err != nil { d.logError(err.Error(), "get file "+path+" for pod "+pod.Name) log.Printf("Error: get %s file: %v", path, err) @@ -426,7 +426,10 @@ func (d *Dumper) DumpCluster() error { // runCmd run command (Dumper.cmd) with given args, return it output func (d *Dumper) runCmd(args ...string) ([]byte, error) { - baseArgs := []string{"--kubeconfig", d.kubeconfig} + var baseArgs []string + if d.kubeconfig != "" { + baseArgs = []string{"--kubeconfig", d.kubeconfig} + } baseArgs = append(baseArgs, args...) var outb, errb bytes.Buffer @@ -524,18 +527,16 @@ func (d *Dumper) parseEnvs(namespace, podName, env string) (string, error) { "sh", "-c", fmt.Sprintf("echo %s", env), } - fmt.Printf("DBG: running parseEnvs: %v\n", args) out, err := d.runCmd(args...) if err != nil { return "", err } - fmt.Printf("DBG: running parseEnvs output: %v\n", string(out)) resolved := strings.TrimSpace(string(out)) return resolved, nil } -func (d *Dumper) getAllFilesFormDirectory(namespace string, podName, path, location string, tw *tar.Writer) error { +func (d *Dumper) getAllFilesFromDirectory(namespace string, podName, path, location string, tw *tar.Writer) error { if len(d.fileContainer) == 0 { return errors.Errorf("Logs container name is not specified for resource %s in namespace %s", resourceType(d.crType), d.namespace) } @@ -675,6 +676,7 @@ func (d *Dumper) getPodSummary(resource, podName, crName string, namespace strin cmdPortFwd := exec.Command(d.cmd, argPortFwd...) err := cmdPortFwd.Start() if err != nil { + d.logError(err.Error(), "start port-forward") return nil, err } From ef03279f2e4b780599253ce13badd08533891620 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 8 Jan 2026 14:59:27 +0200 Subject: [PATCH 03/21] PT-2421 - full refactor of a pt-k8s-debug-collector This refactor includes replacing all of the `kubectl` cli calls with golang sdk for k8s. Additionaly dumper now has new structure, new logger, tar file path controll, and multithreaded approach for downloading and exporting files form multiple pods. --- go.mod | 95 +- go.sum | 229 +++-- .../pt-k8s-debug-collector/dumper/dumper.go | 902 ++++++------------ .../dumper/dumper_test.go | 7 +- .../dumper/individual_files.go | 69 ++ .../dumper/kube_utils.go | 135 +++ .../pt-k8s-debug-collector/dumper/logger.go | 45 + src/go/pt-k8s-debug-collector/dumper/logs.go | 62 ++ src/go/pt-k8s-debug-collector/dumper/paths.go | 41 + .../dumper/resources.go | 120 +++ .../pt-k8s-debug-collector/dumper/secrets.go | 126 +++ .../pt-k8s-debug-collector/dumper/summary.go | 121 +++ src/go/pt-k8s-debug-collector/dumper/tar.go | 66 ++ src/go/pt-k8s-debug-collector/main.go | 8 +- src/go/pt-k8s-debug-collector/main_test.go | 63 +- src/go/tests/utils/anydbver.go | 51 + 16 files changed, 1449 insertions(+), 691 deletions(-) create mode 100644 src/go/pt-k8s-debug-collector/dumper/individual_files.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/kube_utils.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/logger.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/logs.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/paths.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/resources.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/secrets.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/summary.go create mode 100644 src/go/pt-k8s-debug-collector/dumper/tar.go create mode 100644 src/go/tests/utils/anydbver.go diff --git a/go.mod b/go.mod index f1c7381fa..988ff414f 100644 --- a/go.mod +++ b/go.mod @@ -1,75 +1,114 @@ module github.com/percona/percona-toolkit -go 1.25.5 +go 1.24.0 + +toolchain go1.24.1 require ( github.com/AlekSi/pointer v1.2.0 github.com/Ladicle/tabwriter v1.0.0 github.com/Masterminds/semver v1.5.0 github.com/alecthomas/kingpin v2.2.6+incompatible - github.com/alecthomas/kong v1.13.0 + github.com/alecthomas/kong v1.12.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/go-ini/ini v1.67.0 + github.com/goforj/godump v1.9.0 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-version v1.8.0 + github.com/hashicorp/go-version v1.7.0 github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef github.com/lib/pq v1.10.9 github.com/mattn/go-shellwords v1.0.12 github.com/montanaflynn/stats v0.7.1 github.com/pborman/getopt v1.1.0 - github.com/percona/go-mysql v0.0.0-20251202083530-b3e1c16efc74 + github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.34.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 github.com/xlab/treeprint v1.2.0 - go.mongodb.org/mongo-driver v1.17.6 - golang.org/x/crypto v0.45.0 - golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39 + go.mongodb.org/mongo-driver v1.17.4 + go.yaml.in/yaml/v2 v2.4.2 + golang.org/x/crypto v0.43.0 + golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.34.2 - k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + k8s.io/api v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 ) require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/cilium/ebpf v0.20.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/cosiner/argv v0.1.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect + github.com/derekparker/trie/v3 v3.2.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-delve/delve v1.26.0 // indirect + github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/snappy v1.0.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-dap v0.12.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.2 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/tklauser/go-sysconf v0.3.16 // indirect - github.com/tklauser/numcpus v0.11.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.2.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect + go.starlark.net v0.0.0-20260102030733-3fee463870c9 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/telemetry v0.0.0-20251222180846-3f2a21fb04ff // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.9.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apimachinery v0.34.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect - sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.1 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index ee3e1fdc9..4cc56d0a8 100644 --- a/go.sum +++ b/go.sum @@ -8,68 +8,124 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= -github.com/alecthomas/kong v1.13.0 h1:5e/7XC3ugvhP1DQBmTS+WuHtCbcv44hsohMgcvVxSrA= -github.com/alecthomas/kong v1.13.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= -github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= -github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/kong v1.12.1 h1:iq6aMJDcFYP9uFrLdsiZQ2ZMmcshduyGv4Pek0MQPW0= +github.com/alecthomas/kong v1.12.1/go.mod h1:p2vqieVMeTAnaC83txKtXe8FLke2X07aruPWXyMPQrU= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= -github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/cilium/ebpf v0.20.0 h1:atwWj9d3NffHyPZzVlx3hmw1on5CLe9eljR8VuHTwhM= +github.com/cilium/ebpf v0.20.0/go.mod h1:pzLjFymM+uZPLk/IXZUL63xdx5VXEo+enTzxkZXdycw= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= +github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= +github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/derekparker/trie/v3 v3.2.1 h1:fkW2422T+lmRCKD7zuQ97MPgKETXkmJGa0Uze3+5nfU= +github.com/derekparker/trie/v3 v3.2.1/go.mod h1:P94lW0LPgiaMgKAEQD59IDZD2jMK9paKok8Nli/nQbE= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-delve/delve v1.26.0 h1:YZT1kXD76mxba4/wr+tyUa/tSmy7qzoDsmxutT42PIs= +github.com/go-delve/delve v1.26.0/go.mod h1:8BgFFOXTi1y1M+d/4ax1LdFw0mlqezQiTZQpbpwgBxo= +github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA= +github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= -github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/goforj/godump v1.9.0 h1:Y/APfWKQKnJetXgVJxDqD7vEpTGSgAwbKJGmj0UAteI= +github.com/goforj/godump v1.9.0/go.mod h1:/Vy+p50JtOkwsFN5dA1HQ7LS5gtPk3f61DaP4UR2o4s= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM= +github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= -github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -78,48 +134,66 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pborman/getopt v1.1.0 h1:eJ3aFZroQqq0bWmraivjQNt6Dmm5M0h2JcDW38/Azb0= github.com/pborman/getopt v1.1.0/go.mod h1:FxXoW1Re00sQG/+KIkuSqRL/LwQgSkv7uyac+STFsbk= -github.com/percona/go-mysql v0.0.0-20251202083530-b3e1c16efc74 h1:uULelXfIrpmKtIfPjYHnC8TlnQvqza7BT+DGdxYcAZk= -github.com/percona/go-mysql v0.0.0-20251202083530-b3e1c16efc74/go.mod h1:8n6mF3Igr3YYdh19ZUvyVyEH+hQr9LVSjG9hrlp0YTU= +github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c h1:1SZ7nS+kSaO63IpaKspf/gf8602QcgP2eXNPMNOIc0M= +github.com/percona/go-mysql v0.0.0-20210427141028-73d29c6da78c/go.mod h1:/SGLf9OMxlnK6jq4mkFiImBcJXXk5jwD+lDrwDaGXcw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= -github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= -github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= -github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= -github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= @@ -130,20 +204,32 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= -go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= +go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.starlark.net v0.0.0-20260102030733-3fee463870c9 h1:nV1OyvU+0CYrp5eKfQ3rD03TpFYYhH08z31NK1HmtTk= +go.starlark.net v0.0.0-20260102030733-3fee463870c9/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= +golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= -golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39 h1:DHNhtq3sNNzrvduZZIiFyXWOL9IWaDPHqTnLJp+rCBY= -golang.org/x/exp v0.0.0-20251125195548-87e1e737ad39/go.mod h1:46edojNIoXTNOhySWIWdix628clX9ODXwPsQuG6hsK0= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -155,15 +241,21 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -172,60 +264,83 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20251222180846-3f2a21fb04ff h1:1QaeZGjxSnF1KOGnUYQmI1YpaBe0FvBE1K2rRDuxawc= +golang.org/x/telemetry v0.0.0-20251222180846-3f2a21fb04ff/go.mod h1:ArQvPJS723nJQietgilmZA+shuB3CZxH1n2iXq9VSfs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= -k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1 h1:JrhdFMqOd/+3ByqlP2I45kTOZmTRLBUm5pvRjeheg7E= -sigs.k8s.io/structured-merge-diff/v6 v6.3.1/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index a5e2958e3..1cbe851aa 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -1,708 +1,424 @@ package dumper import ( - "archive/tar" - "bytes" - "compress/gzip" - "encoding/base64" - "encoding/json" + "context" + "fmt" + "io" "log" - "os" - "os/exec" "path/filepath" - "regexp" "strings" - "text/template" + "sync" "time" - "github.com/pkg/errors" + "go.yaml.in/yaml/v2" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" ) -// sslSecret struct for dumping certificates -type sslSecret struct { - secret string - resource string - dataNames []string -} +const ( + NumWorkers = 10 // Concurrency for Pod Logs +) // Dumper struct is for dumping cluster type Dumper struct { - cmd string kubeconfig string - resources []string - filePaths []string - fileContainer string namespace string location string - errors string + logger *SafeLogger mode int64 - crType string + crTypes []string forwardport string - sslSecrets []sslSecret skipPodSummary bool -} -var resourcesRe = regexp.MustCompile(`(\w+\.(\w+).percona\.com)`) - -// New return new Dumper object -func New(location, namespace, resource string, kubeconfig string, forwardport string, skipPodSummary bool) Dumper { - d := Dumper{ - cmd: "kubectl", - kubeconfig: kubeconfig, - location: "cluster-dump", - mode: int64(0o777), - namespace: namespace, - forwardport: forwardport, - skipPodSummary: skipPodSummary, - } - resources := []string{ - "pods", - "replicasets", - "deployments", - "statefulsets", - "replicationcontrollers", - "events", - "configmaps", - "cronjobs", - "jobs", - "poddisruptionbudgets", - "clusterrolebindings", - "clusterroles", - "rolebindings", - "roles", - "storageclasses", - "persistentvolumeclaims", - "persistentvolumes", - } + sslSecrets map[string]bool + individualFiles []individualFile + clientSet *kubernetes.Clientset + dynamicClient *dynamic.DynamicClient + discoveryClient *discovery.DiscoveryClient + archive *tarWriter + restConfig *rest.Config +} - switch resourceType(resource) { - case "auto": - result, err := d.runCmd("api-resources", "-o", "name") - if err != nil { - log.Panicf("Cannot get API resources and option --resource=auto specified:\n%s", err) - } - matches := resourcesRe.FindAllStringSubmatch(string(result), -1) - if len(matches) == 0 { - resource = "none" - break - } - for _, match := range matches { - resources = append(resources, match[1]) - resource = match[2] - } - case "pg": - resources = append(resources, - "perconapgclusters.pg.percona.com", - "pgclusters.pg.percona.com", - "pgpolicies.pg.percona.com", - "pgreplicas.pg.percona.com", - "pgtasks.pg.percona.com", - ) - case "pgv2": - resources = append(resources, - "perconapgbackups.pgv2.percona.com", - "perconapgclusters.pgv2.percona.com", - "perconapgrestores.pgv2.percona.com", - ) - case "pxc": - resources = append(resources, - "perconaxtradbclusterbackups.pxc.percona.com", - "perconaxtradbclusterrestores.pxc.percona.com", - "perconaxtradbclusters.pxc.percona.com", - ) - case "ps": - resources = append(resources, - "perconaservermysqlbackups.ps.percona.com", - "perconaservermysqlrestores.ps.percona.com", - "perconaservermysqls.ps.percona.com", - ) - case "psmdb": - resources = append(resources, - "perconaservermongodbbackups.psmdb.percona.com", - "perconaservermongodbrestores.psmdb.percona.com", - "perconaservermongodbs.psmdb.percona.com", - ) - } - sslSecrets := make([]sslSecret, 0) - filePaths := make([]string, 0) - switch resourceType(resource) { - case "pg": - sslSecrets = append(sslSecrets, - sslSecret{ - secret: "{{ .Name }}-ssl-ca", - resource: "perconapgclusters.pg.percona.com", - dataNames: []string{"ca.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-ssl-keypair", - resource: "perconapgclusters.pg.percona.com", - dataNames: []string{"tls.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-replication-ssl-keypair", - resource: "perconapgclusters.pg.percona.com", - dataNames: []string{"tls.crt"}, - }, - sslSecret{ - secret: "pgo.tls", - resource: "perconapgclusters.pg.percona.com", - dataNames: []string{"tls.crt"}, - }, - ) - case "pgv2": - sslSecrets = append(sslSecrets, - sslSecret{ - secret: "{{ .Name }}-cluster-cert", - resource: "pg", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - sslSecret{ - secret: "pgo-root-cacert", - resource: "pg", - dataNames: []string{"root.crt"}, - }, - ) - case "pxc": - filePaths = append(filePaths, - "var/lib/mysql/mysqld-error.log", - "var/lib/mysql/innobackup.backup.log", - "var/lib/mysql/innobackup.move.log", - "var/lib/mysql/innobackup.prepare.log", - "var/lib/mysql/grastate.dat", - "var/lib/mysql/gvwstate.dat", - "var/lib/mysql/mysqld.post.processing.log", - "var/lib/mysql/auto.cnf", - ) - d.fileContainer = "logs" - sslSecrets = append(sslSecrets, - sslSecret{ - secret: "{{ .Name }}-ssl", - resource: "pxc", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-ssl-internal", - resource: "pxc", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-ca-cert", - resource: "pxc", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - ) - case "ps": - sslSecrets = append(sslSecrets, - sslSecret{ - secret: "{{ .Name }}-ssl", - resource: "ps", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-ca-cert", - resource: "ps", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - ) - case "psmdb": - sslSecrets = append(sslSecrets, - sslSecret{ - secret: "{{ .Name }}-ssl", - resource: "psmdb", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-ssl-internal", - resource: "psmdb", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - sslSecret{ - secret: "{{ .Name }}-ca-cert", - resource: "psmdb", - dataNames: []string{"ca.crt", "tls.crt"}, - }, - ) - } - d.resources = resources - d.sslSecrets = sslSecrets - d.crType = resource - d.filePaths = filePaths - return d +// individualFile struct is used to dump the necessary files from the containers +type individualFile struct { + resourceName string + containerName string + filepaths []string } -type k8sPods struct { - Items []corev1.Pod `json:"items"` +// resourceMap struct is used to dump the resources from namespace scope or cluster scope +type resourceMap struct { + ClusterScoped []schema.GroupVersionResource + NamespaceScoped []schema.GroupVersionResource } -type namespaces struct { - Items []corev1.Namespace `json:"items"` +// exportJob struct is used in goroutines to access pods +type exportJob struct { + Pod corev1.Pod } -// DumpCluster create dump of a cluster in Dumper.location -func (d *Dumper) DumpCluster() error { - file, err := os.Create(d.location + ".tar.gz") - if err != nil { - return errors.Wrap(err, "create tar file") - } +// New return new Dumper object +func New(location, namespace, kubeconfig, forwardport, resource string, skipPodSummary bool) (*Dumper, error) { + var ( + err error + config *rest.Config + ) - zr := gzip.NewWriter(file) - tw := tar.NewWriter(zr) defer func() { - err = addToArchive(d.location+"/errors.txt", d.mode, []byte(d.errors), tw) - if err != nil { - log.Println("Error: add errors.txt to archive:", err) - } - - err = tw.Close() - if err != nil { - log.Println("close tar writer", err) - return - } - err = zr.Close() - if err != nil { - log.Println("close gzip writer", err) - return - } - err = file.Close() - if err != nil { - log.Println("close file", err) - return + if r := recover(); r != nil { + err = fmt.Errorf("recovered from panic: %s", r) } }() - var nss namespaces + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("failed to build config from flags: %w", err) + } - if len(d.namespace) > 0 { - ns := corev1.Namespace{} - ns.Name = d.namespace - nss.Items = append(nss.Items, ns) - } else { - args := []string{"get", "namespaces", "-o", "json"} - output, err := d.runCmd(args...) - if err != nil { - d.logError(err.Error(), args...) - return errors.Wrap(err, "get namespaces") - } + config.QPS = 10 + config.Burst = 20 + + clientset := kubernetes.NewForConfigOrDie(config) + dynclient := dynamic.NewForConfigOrDie(config) + discclient := discovery.NewDiscoveryClientForConfigOrDie(config) + + // TODO: implement cluster name flag + d := &Dumper{ + kubeconfig: kubeconfig, + location: location, + mode: int64(0o777), + namespace: namespace, + forwardport: forwardport, + skipPodSummary: skipPodSummary, + clientSet: clientset, + dynamicClient: dynclient, + discoveryClient: discclient, + restConfig: config, + logger: NewSafeLogger(), + } - err = json.Unmarshal(output, &nss) - if err != nil { - d.logError(err.Error(), "unmarshal namespaces") - return errors.Wrap(err, "unmarshal namespaces") - } + if resource == "none" || resource == "" { + return d, nil } - for _, ns := range nss.Items { - args := []string{"get", "pods", "-o", "json", "--namespace", ns.Name} - output, err := d.runCmd(args...) - if err != nil { - d.logError(err.Error(), args...) - continue - } + d.crTypes, err = d.autoCustomResource() + if err != nil { + return nil, err + } - var pods k8sPods - err = json.Unmarshal(output, &pods) - if err != nil { - d.logError(err.Error(), "unmarshal pods from namespace", ns.Name) - log.Printf("Error: unmarshal pods in namespace %s: %v", ns.Name, err) - } + log.Printf("DBG: FOUND CR's: %v", d.crTypes) - for _, pod := range pods.Items { - location := filepath.Join(d.location, ns.Name, pod.Name, "logs.txt") - args := []string{"logs", pod.Name, "--namespace", ns.Name, "--all-containers"} - output, err = d.runCmd(args...) + d.sslSecrets = make(map[string]bool, 0) + for _, cr := range d.crTypes { + switch resourceType(cr) { + case "pg": + err := d.addPg1() if err != nil { - d.logError(err.Error(), args...) - err = addToArchive(location, d.mode, []byte(err.Error()), tw) - if err != nil { - log.Printf("Error: create archive with logs for pod %s in namespace %s: %v", pod.Name, ns.Name, err) - } - continue + return nil, err } - err = addToArchive(location, d.mode, output, tw) + case "pgv2": + err := d.addPg2() if err != nil { - d.logError(err.Error(), "create archive for pod "+pod.Name) - log.Printf("Error: create archive for pod %s: %v", pod.Name, err) - } - if len(pod.Labels) == 0 { - continue - } - location = filepath.Join(d.location, ns.Name, pod.Name, "/summary.txt") - component := resourceType(d.crType) - if component == "psmdb" { - component = "mongod" - } - if component == "ps" { - component = "mysql" - } - if pod.Labels["app.kubernetes.io/instance"] != "" && pod.Labels["app.kubernetes.io/component"] != "" { - resource := "secret/" + pod.Labels["app.kubernetes.io/instance"] + "-" + pod.Labels["app.kubernetes.io/component"] - err = d.getResource(resource, ns.Name, true, tw) - if err != nil { - log.Printf("Error: get %s resource: %v", resource, err) - } + return nil, err } - if pod.Labels["app.kubernetes.io/component"] == component || - (component == "pg" && pod.Labels["pgo-pg-database"] == "true") || - (component == "pgv2" && pod.Labels["pgv2.percona.com/version"] != "" && pod.Labels["postgres-operator.crunchydata.com/instance"] != "") { - var crName string - if component == "pg" { - crName = pod.Labels["pg-cluster"] - } else if component == "pgv2" { - crName = pod.Labels["postgres-operator.crunchydata.com/cluster"] - } else { - crName = pod.Labels["app.kubernetes.io/instance"] - } - // Get summary - if !d.skipPodSummary { - output, err = d.getPodSummary(resourceType(d.crType), pod.Name, crName, ns.Name) - if err != nil { - d.logError(err.Error(), d.crType, pod.Name) - err = addToArchive(location, d.mode, []byte(err.Error()), tw) - if err != nil { - log.Printf("Error: create summary errors archive for pod %s in namespace %s: %v", pod.Name, ns.Name, err) - } - } else { - err = addToArchive(location, d.mode, output, tw) - if err != nil { - d.logError(err.Error(), "create summary archive for pod "+pod.Name) - log.Printf("Error: create summary archive for pod %s: %v", pod.Name, err) - } - } - } - - // get individual Logs - location = filepath.Join(d.location, ns.Name, pod.Name) - for _, path := range d.filePaths { - err = d.getIndividualFiles(ns.Name, pod.Name, path, location, tw) - if err != nil { - d.logError(err.Error(), "get file "+path+" for pod "+pod.Name) - log.Printf("Error: get %s file: %v", path, err) - } - } + case "pxc": + err := d.addPxc() + if err != nil { + return nil, err } - } - - for _, resource := range d.resources { - err = d.getResource(resource, ns.Name, false, tw) + case "ps": + err := d.addPs() if err != nil { - log.Printf("Error: get %s resource: %v", resource, err) + return nil, err } - } - - for _, s := range d.sslSecrets { - err = d.getSSLCertificates(s, ns.Name, tw) + case "psmdb": + err := d.addPsmdb() if err != nil { - log.Printf("Error: get SSL certificates in %s: %v", s.secret, err) + return nil, err } } } - err = d.getResource("nodes", "", false, tw) + return d, err +} + +// DumpCluster create dump of a cluster in Dumper.location +func (d *Dumper) DumpCluster() error { + var err error + d.archive, err = NewTarWriter(d.location + ".tar.gz") if err != nil { - return errors.Wrapf(err, "get nodes") + return fmt.Errorf("Failed to create archive: %v", err) } + defer d.archive.Close() - return nil -} + oldLoggerOut := log.Writer() + log.SetOutput(io.MultiWriter(log.Writer(), d.logger)) + defer log.SetOutput(oldLoggerOut) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + defer d.logger.DumpToArchive(d.archive, d.DumperLogPath("dumper")) -// runCmd run command (Dumper.cmd) with given args, return it output -func (d *Dumper) runCmd(args ...string) ([]byte, error) { - var outb, errb bytes.Buffer - args = append(args, "--kubeconfig", d.kubeconfig) - cmd := exec.Command(d.cmd, args...) - cmd.Stdout = &outb - cmd.Stderr = &errb - err := cmd.Run() - if err != nil || errb.Len() > 0 { - return nil, errors.Errorf("error: %v, stderr: %s, stdout: %s", err, errb.String(), outb.String()) + log.Println("Initializing Pod Cache") + factory := informers.NewSharedInformerFactory(d.clientSet, 10*time.Minute) + podInformer := factory.Core().V1().Pods().Informer() + factory.Start(ctx.Done()) + + log.Println("Discovering and Exporting API Resources") + if err := d.export(ctx); err != nil { + log.Printf("Error during resource export: %v", err) } - return outb.Bytes(), nil -} + log.Println("Starting Workers for Pod Logs/Files...") + jobsChannel := make(chan exportJob, 100) + var wg sync.WaitGroup -func (d *Dumper) getResource(name, namespace string, ignoreNotFound bool, tw *tar.Writer) error { - location := d.location - args := []string{"get", name, "-o", "yaml"} - if ignoreNotFound { - args = append(args, "--ignore-not-found") + for i := range NumWorkers { + wg.Add(1) + go func(id int) { + defer wg.Done() + d.resilientWorker(id, ctx, cancel, jobsChannel) + }(i) } - if len(namespace) > 0 { - args = append(args, "--namespace", namespace) - location = filepath.Join(d.location, namespace) + + log.Println("Waiting for Pod Cache to fully sync...") + if !cache.WaitForCacheSync(ctx.Done(), podInformer.HasSynced) { + return fmt.Errorf("Timed out waiting for caches to sync.") } - location = filepath.Join(location, name+".yaml") - output, err := d.runCmd(args...) + + podLister := factory.Core().V1().Pods().Lister() + allPods, err := podLister.List(labels.Everything()) if err != nil { - d.logError(err.Error(), args...) - log.Printf("Error: get resource %s in namespace %s: %v", name, namespace, err) - return addToArchive(location, d.mode, []byte(err.Error()), tw) + return fmt.Errorf("Failed to list all pods: %v", err) } - if ignoreNotFound && len(output) == 0 { - return nil + log.Printf("Dispatching %d pods to workers...", len(allPods)) + for _, pod := range allPods { + jobsChannel <- exportJob{Pod: *pod} } - return addToArchive(location, d.mode, output, tw) -} -func (d *Dumper) logError(err string, args ...string) { - d.errors += d.cmd + " " + strings.Join(args, " ") + "\n" + err + "\n\n" -} - -func addToArchive(location string, mode int64, content []byte, tw *tar.Writer) error { - hdr := &tar.Header{ - Name: location, - Mode: mode, - ModTime: time.Now(), - Size: int64(len(content)), - } - if err := tw.WriteHeader(hdr); err != nil { - return errors.Wrapf(err, "write header to %s", location) - } - if _, err := tw.Write(content); err != nil { - return errors.Wrapf(err, "write content to %s", location) - } + close(jobsChannel) + wg.Wait() + log.Printf("Export Complete. Data saved to %s", d.location) return nil } -type crSecrets struct { - Spec struct { - SecretName string `json:"secretsName,omitempty"` - Secrets struct { - Users string `json:"users,omitempty"` - } `json:"secrets,omitempty"` - Users []struct { - Name string `json:"name,omitempty"` - SecretName string `json:"secretName,omitempty"` - } `json:"users,omitempty"` - } `json:"spec"` -} - -func (d *Dumper) getIndividualFiles(namespace string, podName, path, location string, tw *tar.Writer) error { - if len(d.fileContainer) == 0 { - return errors.Errorf("Logs container name is not specified for resource %s in namespace %s", resourceType(d.crType), d.namespace) - } - args := []string{"-n", namespace, "-c", d.fileContainer, "cp", podName + ":" + path, "/dev/stdout"} - output, err := d.runCmd(args...) +func (d *Dumper) export(ctx context.Context) error { + resources, err := d.discoverResources() if err != nil { - d.logError(err.Error(), args...) - log.Printf("Error: get path %s for resource %s in namespace %s: %v", path, resourceType(d.crType), d.namespace, err) - return addToArchive(location, d.mode, []byte(err.Error()), tw) + return err } - if len(output) == 0 { - return nil - } - return addToArchive(location+"/"+path, d.mode, output, tw) -} + var wg sync.WaitGroup + semCluster := make(chan struct{}, 5) -func (d *Dumper) getPodSummary(resource, podName, crName string, namespace string) ([]byte, error) { - var ( - summCmdName string - ports string - summCmdArgs []string - ) + for _, gvr := range resources.ClusterScoped { + wg.Add(1) + go func(r schema.GroupVersionResource) { + defer wg.Done() + semCluster <- struct{}{} + defer func() { <-semCluster }() - switch resource { - case "ps": - fallthrough - case "pxc": - var pass, port string - if d.forwardport != "" { - port = d.forwardport - } else { - port = "3306" - } - cr, err := d.getCR(resource+"/"+crName, namespace) - if err != nil { - return nil, errors.Wrap(err, "get cr") - } - if cr.Spec.SecretName != "" { - pass, err = d.getDataFromSecret(cr.Spec.SecretName, "root", namespace) - } else { - pass, err = d.getDataFromSecret(crName+"-secrets", "root", namespace) - } - if err != nil { - return nil, errors.Wrap(err, "get password from pxc users secret") - } - ports = port + ":3306" - summCmdName = "pt-mysql-summary" - summCmdArgs = []string{"--host=127.0.0.1", "--port=" + port, "--user=root", "--password=" + string(pass)} - case "pg", "pgv2": - var kubeconfig string - if d.kubeconfig != "" { - kubeconfig = " --kubeconfig=" + d.kubeconfig - } - summCmdName = "sh" - summCmdArgs = []string{"-c", "curl https://raw.githubusercontent.com/percona/support-snippets/master/postgresql/pg_gather/gather.sql 2>/dev/null | " + - d.cmd + kubeconfig + " -n " + namespace + " exec -i " + podName + " -- psql -X -f - "} - case "psmdb": - var port string - if d.forwardport != "" { - port = d.forwardport - } else { - port = "27017" - } - cr, err := d.getCR("psmdb/"+crName, namespace) - if err != nil { - return nil, errors.Wrap(err, "get cr") - } - user, err := d.getDataFromSecret(cr.Spec.Secrets.Users, "MONGODB_DATABASE_ADMIN_USER", namespace) - if err != nil { - return nil, errors.Wrap(err, "get user name from psmdb users secret") - } - pass, err := d.getDataFromSecret(cr.Spec.Secrets.Users, "MONGODB_DATABASE_ADMIN_PASSWORD", namespace) - if err != nil { - return nil, errors.Wrap(err, "get password from psmdb users secret") - } - ports = port + ":27017" - summCmdName = "pt-mongodb-summary" - summCmdArgs = []string{"--username=" + user, "--password=" + string(pass), "--authenticationDatabase=admin", "127.0.0.1:" + port} + err := d.exportGeneric(ctx, r, "") + if err != nil { + log.Printf("failed to export resource %q: %s", r.Resource, err) + return + } + }(gvr) } - cmdPortFwd := exec.Command(d.cmd, "port-forward", "pod/"+podName, ports, "-n", namespace, "--kubeconfig", d.kubeconfig) - go func() { - err := cmdPortFwd.Run() - if err != nil { - d.logError(err.Error(), "port-forward") - } - }() - defer func() { - err := cmdPortFwd.Process.Kill() - if err != nil { - d.logError(err.Error(), "kill port-forward") - } - }() - - time.Sleep(3 * time.Second) // wait for port-forward command - - var outb, errb bytes.Buffer - cmd := exec.Command(summCmdName, summCmdArgs...) - cmd.Stdout = &outb - cmd.Stderr = &errb - err := cmd.Run() + namespaces, err := d.clientSet.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { - return nil, errors.Wrapf(err, "stderr: %s\nstdout: %s", errb.String(), outb.String()) + return err } - return outb.Bytes(), nil -} -func (d *Dumper) getCR(crName string, namespace string) (crSecrets, error) { - var cr crSecrets - output, err := d.runCmd("get", crName, "-o", "json", "-n", namespace) - if err != nil { - return cr, errors.Wrap(err, "get "+crName) - } - err = json.Unmarshal(output, &cr) - if err != nil { - return cr, errors.Wrap(err, "unmarshal "+crName+" cr") + semNS := make(chan struct{}, 5) + + for _, ns := range namespaces.Items { + if d.namespace != "" && d.namespace != ns.Name { + continue + } + wg.Add(1) + go func(namespace string) { + defer wg.Done() + semNS <- struct{}{} + defer func() { <-semNS }() + + if err := d.dumpSecrets(ctx, namespace); err != nil { + log.Printf("Error dumping secrets for %s: %v", namespace, err) + } + + for _, gvr := range resources.NamespaceScoped { + if gvr.Resource == "secrets" { + continue + } + d.exportGeneric(ctx, gvr, namespace) + } + }(ns.Name) } - return cr, nil + wg.Wait() + return nil } -func (d *Dumper) getDataFromSecret(secretName, dataName string, namespace string) (string, error) { - passEncoded, err := d.runCmd("get", "secrets/"+secretName, "--template={{.data."+dataName+"}}", "-n", namespace) - if err != nil { - return "", errors.Wrap(err, "run get secret cmd") +func (d *Dumper) exportGeneric(ctx context.Context, gvr schema.GroupVersionResource, ns string) error { + var intf dynamic.ResourceInterface + if ns == "" { + intf = d.dynamicClient.Resource(gvr) + } else { + intf = d.dynamicClient.Resource(gvr).Namespace(ns) } - pass, err := base64.StdEncoding.DecodeString(string(passEncoded)) - if err != nil { - return "", errors.Wrap(err, "decode data") + + list, err := intf.List(ctx, metav1.ListOptions{}) + if err != nil || len(list.Items) == 0 { + return err } - return string(pass), nil -} + for i := range list.Items { + obj := list.Items[i].Object + if meta, ok := obj["metadata"].(map[string]interface{}); ok { + delete(meta, "managedFields") + delete(meta, "resourceVersion") + delete(meta, "uid") + delete(meta, "selfLink") + delete(meta, "creationTimestamp") + } + } -func (d *Dumper) getSSLDataFromSecret(secretName, dataName string, namespace string) (string, error) { - data, err := d.runCmd("get", "secrets/"+secretName, "-o", "go-template='{{ index .data \""+dataName+"\" | base64decode }}'", "-n", namespace) + data, err := yaml.Marshal(list.UnstructuredContent()) if err != nil { - return "", errors.Wrap(err, "run get secret cmd") + return err } - return string(data), nil -} + yamlPath := d.PodResourcePath(ns, gvr.Resource) + d.archive.WriteVirtualFile(yamlPath, data) -func (d *Dumper) getSSLCertificates(secret sslSecret, namespace string, tw *tar.Writer) error { - cr := struct { - Items []struct { - Metadata struct { - Name string `json:"name"` - } `json:"metadata"` - } `json:"items"` - }{} + return nil +} - output, err := d.runCmd("get", secret.resource, "-o", "json", "-n", namespace) +func (d *Dumper) discoverResources() (*resourceMap, error) { + lists, err := d.discoveryClient.ServerPreferredResources() if err != nil { - return errors.Wrap(err, "get "+secret.resource) + return nil, err } - err = json.Unmarshal(output, &cr) - if err != nil { - return errors.Wrapf(err, "unmarshal %s cr", secret.resource) + rm := &resourceMap{} + + ignoredResources := map[string]bool{ + "apiaccesses": true, // Deprecated + "componentstatuses": true, // Deprecated + "endpoints": true, // Deprecated + "events": true, // Too noisy + "pods": true, // Handled by workers } - if len(cr.Items) > 1 { - return errors.Wrap(err, "Unexpected structure for resource "+secret.resource) - } + for _, list := range lists { + gv, _ := schema.ParseGroupVersion(list.GroupVersion) + for _, resource := range list.APIResources { + if strings.Contains(resource.Name, "/") { + continue + } - for _, item := range cr.Items { - location := d.location + if ignoredResources[resource.Name] { + continue + } - if len(namespace) > 0 { - location = filepath.Join(d.location, namespace) + gvr := schema.GroupVersionResource{ + Group: gv.Group, + Version: gv.Version, + Resource: resource.Name, + } + if resource.Namespaced { + rm.NamespaceScoped = append(rm.NamespaceScoped, gvr) + } else { + rm.ClusterScoped = append(rm.ClusterScoped, gvr) + } } + } + return rm, nil +} - var nb bytes.Buffer - t := template.Must(template.New("secret").Parse(secret.secret)) - t.Execute(&nb, item.Metadata) - - name := nb.String() - location = filepath.Join(location, name) +func (d *Dumper) resilientWorker(id int, ctx context.Context, cancel context.CancelFunc, jobs <-chan exportJob) { + for { + select { + case <-ctx.Done(): + return + case job, ok := <-jobs: + if !ok { + return + } - result := make([]byte, 0) - for _, dn := range secret.dataNames { - result = append(result, dn+"\n"...) - data, err := d.getSSLDataFromSecret(name, dn, namespace) + err := d.exportPodLogs(ctx, job.Pod) if err != nil { - errors.Wrapf(err, "Error getting certificate %s from secret %s", dn, name) + if isSpaceError(err) { + log.Printf("Worker %d stopping app: %v", id, err) + cancel() + return + } + report := fmt.Sprintf("Error exporting logs: %v", err) + errPath := filepath.Join(d.location, job.Pod.Namespace, job.Pod.Name) + d.archive.WriteVirtualFile(errPath, []byte(report)) } - var outb, errb bytes.Buffer - cmd := exec.Command("sh", "-c", "echo "+data+" | openssl x509 -noout -text") - cmd.Stdout = &outb - cmd.Stderr = &errb - err = cmd.Run() - if err != nil { - errors.Wrapf(err, "stderr: %s\nstdout: %s", errb.String(), outb.String()) + if job.Pod.Status.Phase == corev1.PodRunning { + d.exportPodSummaryAndFiles(ctx, job) } - result = append(result, outb.Bytes()...) } + } +} - err = addToArchive(location, d.mode, result, tw) +var crLabelMap = map[string]string{ + "psmdb": "mongod", + "ps": "mysql", +} - if err != nil { - return errors.Wrapf(err, "Cannot add certificates in the secret %s into resulting archive", name) - } +func matchesCR(cr string, podLabels map[string]string) bool { + if mapped, ok := crLabelMap[cr]; ok { + cr = mapped + } + if podLabels["app.kubernetes.io/component"] == cr || + podLabels["app.kubernetes.io/name"] == cr { + return true } - return nil + switch cr { + case "pg": + return podLabels["pgo-pg-database"] == "true" + case "pgo": + return podLabels["pgo-pg-database"] == "true" + case "pgv2": + return podLabels["pgv2.percona.com/version"] != "" && + podLabels["postgres-operator.crunchydata.com/instance"] != "" + } + + return false } -func resourceType(s string) string { - if s == "auto" { - return "auto" - } else if s == "pxc" || strings.HasPrefix(s, "pxc/") { - return "pxc" - } else if s == "psmdb" || strings.HasPrefix(s, "psmdb/") { - return "psmdb" - } else if s == "pg" || strings.HasPrefix(s, "pg/") { - return "pg" - } else if s == "pgv2" || strings.HasPrefix(s, "pgv2/") { - return "pgv2" - } else if s == "ps" || strings.HasPrefix(s, "ps/") { - return "ps" +func (d *Dumper) exportPodSummaryAndFiles(ctx context.Context, job exportJob) { + for _, cr := range d.crTypes { + if !matchesCR(cr, job.Pod.Labels) { + continue + } + + if !d.skipPodSummary { + d.getSummary(ctx, job, cr, d.PodSummaryPath(job.Pod.Namespace, job.Pod.Name)) + } + + d.getIndividualFiles(ctx, job, cr) } - return s +} + +func isSpaceError(err error) bool { + return strings.Contains(err.Error(), "no space left on device") } diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper_test.go b/src/go/pt-k8s-debug-collector/dumper/dumper_test.go index d540e1e72..40b901665 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper_test.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper_test.go @@ -11,9 +11,12 @@ Unit test for non-existing logs container name error handling */ func TestGetIndividualFilesError(t *testing.T) { - d := New("", "", "psmdb", "", "") + _, err := New("", "", "psmdb", "", "", false) + if err != nil { + t.Fatalf("failed to create new dumper: %v", err) + } - err := d.getIndividualFiles("", "", "", "", nil) + //err = d.getIndividualFiles("", "", "", "", nil) assert.Error(t, err) assert.ErrorContains(t, err, "Logs container name is not specified") diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go new file mode 100644 index 000000000..66a141978 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -0,0 +1,69 @@ +package dumper + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + + corev1 "k8s.io/api/core/v1" +) + +func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType string) { + for _, indf := range d.individualFiles { + if indf.resourceName == crType { + for _, indPath := range indf.filepaths { + file, err := d.getFileFromPod(ctx, job.Pod, indPath, indf.containerName) + if err != nil { + log.Printf("error while getting individual files for %q pod and %q namespace to dump: %s, SKIPPING", job.Pod.Name, job.Pod.Namespace, err) + continue + } + + if len(file) != 0 { + log.Printf("Writing individual file with path %s to dump", indPath) + path := d.PodIndividualFilesPath(job.Pod.Namespace, job.Pod.Name, indPath) + err = d.archive.WriteVirtualFile(path, file) + if err != nil { + log.Printf("error while writing individual files for %q pod and %q namespace to dump: %s", job.Pod.Name, job.Pod.Namespace, err) + } + } + } + } + } +} + +func (d *Dumper) getFileFromPod(ctx context.Context, pod corev1.Pod, filepath, containerName string) ([]byte, error) { + if len(filepath) == 0 || len(containerName) == 0 { + return nil, errors.New("container name or filepath is not specified") + } + + cmd := []string{"tar", "cf", "-", filepath} + stdout, stderr, err := d.executeInPod(ctx, cmd, pod, containerName, nil) + if err != nil { + return nil, fmt.Errorf("failed to execute command in Pod: stderr: %s: %w", &stderr, err) + } + + tarReader := tar.NewReader(&stdout) + var fileContentBuffer bytes.Buffer + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("error reading tar header: %w", err) + } + + if header.Typeflag == tar.TypeReg && header.Name == filepath { + _, copyErr := io.Copy(&fileContentBuffer, tarReader) + if copyErr != nil { + return nil, fmt.Errorf("error copying file content: %w", copyErr) + } + } + } + + return fileContentBuffer.Bytes(), nil +} diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go new file mode 100644 index 000000000..f7c12ebd9 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -0,0 +1,135 @@ +package dumper + +import ( + "bytes" + "context" + "fmt" + "io" + "log" + "net/http" + "net/url" + "path" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/client-go/transport/spdy" +) + +/* +Forwards ports to a specific pod. Close the returned channel to stop forwarding. +*/ +func (d *Dumper) portForwardPod(ctx context.Context, pod corev1.Pod, remotePort string) (int, error) { + apiURL, err := url.Parse(d.restConfig.Host) + if err != nil { + return 0, fmt.Errorf("failed to parse config host URL: %w", err) + } + + urlPath := path.Join(apiURL.Path, "/api/v1/namespaces/", pod.Namespace, "/pods/", pod.Name, "/portforward") + hostURL := url.URL{ + Scheme: apiURL.Scheme, + Host: apiURL.Host, + Path: urlPath, + } + + roundTripper, upgrader, err := spdy.RoundTripperFor(d.restConfig) + if err != nil { + return 0, fmt.Errorf("failed to create roundtripper and upgrader: %w", err) + } + dialer := spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &hostURL) + readyChan := make(chan struct{}, 1) + out, errOut := new(bytes.Buffer), new(bytes.Buffer) + + var ports []string + if strings.Contains(remotePort, ":") { + ports = []string{remotePort} + } else { + ports = []string{fmt.Sprintf(":%s", remotePort)} + } + + forwarderCtx, forwarderClose := context.WithCancel(ctx) + forwarder, err := portforward.New(dialer, ports, forwarderCtx.Done(), readyChan, out, errOut) + if err != nil { + forwarderClose() + return 0, fmt.Errorf("failed to create port forwarder: %w", err) + } + + go func() { + if err = forwarder.ForwardPorts(); err != nil { + log.Printf("Port forwarding failed: %v", err) + } + forwarderClose() + }() + + select { + case <-readyChan: + forwardedPorts, err := forwarder.GetPorts() + if err != nil { + return 0, fmt.Errorf("failed to get forwarded ports: %w", err) + } + if len(forwardedPorts) == 0 { + return 0, fmt.Errorf("no ports were forwarded") + } + + localPort := int(forwardedPorts[0].Local) + + return localPort, nil + case <-forwarderCtx.Done(): + return 0, fmt.Errorf("port forward stopped unexpectedly before being ready: %s", errOut.String()) + } +} + +/* +Executes a command in the pod and returns the standard output and error streams. +The argument 'stdin' is used for piping and can be set to nil if not required. + +an example of command is: command := []string{"psql", "-X", "-f", "-"} + +The container can be an empty string, but the following rules will be applied: + +1. If the pod has only one container, the command will be executed in that container. + +2. If the pod has multiple containers, it will attempt to use the +container specified by the 'kubectl.kubernetes.io/default-container' annotation on the pod, if present. + +3. If no annotation is present and there are multiple containers, this will return an error. +*/ +func (d *Dumper) executeInPod(ctx context.Context, command []string, pod corev1.Pod, container string, stdin io.Reader) (bytes.Buffer, bytes.Buffer, error) { + stdinFlag := false + if stdin != nil { + stdinFlag = true + } + var outb, errb bytes.Buffer + req := d.clientSet.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdin: stdinFlag, + Stdout: true, + Stderr: true, + TTY: false, + Container: container, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(d.restConfig, "POST", req.URL()) + if err != nil { + return outb, errb, fmt.Errorf("error creating SPDY executor: %w", err) + } + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: &outb, + Stderr: &errb, + Tty: false, + }) + if err != nil { + return outb, errb, fmt.Errorf("error executing remote command: %w", err) + } + + return outb, errb, nil +} diff --git a/src/go/pt-k8s-debug-collector/dumper/logger.go b/src/go/pt-k8s-debug-collector/dumper/logger.go new file mode 100644 index 000000000..46883e42c --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/logger.go @@ -0,0 +1,45 @@ +package dumper + +import ( + "bytes" + "sync" +) + +// SafeLogger is a thread-safe io.Writer that accumulates log bytes. +type SafeLogger struct { + mu sync.Mutex + buf bytes.Buffer +} + +func NewSafeLogger() *SafeLogger { return &SafeLogger{} } + +func (s *SafeLogger) Write(p []byte) (int, error) { + s.mu.Lock() + n, err := s.buf.Write(p) + s.mu.Unlock() + return n, err +} + +func (s *SafeLogger) Bytes() []byte { + s.mu.Lock() + b := make([]byte, s.buf.Len()) + copy(b, s.buf.Bytes()) + s.mu.Unlock() + return b +} + +func (s *SafeLogger) String() string { return string(s.Bytes()) } + +func (s *SafeLogger) DumpToArchive(archive *tarWriter, path string) error { + if archive == nil { + return nil + } + + return archive.WriteFile(path, &s.buf, int64(s.buf.Len())) +} + +func (s *SafeLogger) Reset() { + s.mu.Lock() + s.buf.Reset() + s.mu.Unlock() +} diff --git a/src/go/pt-k8s-debug-collector/dumper/logs.go b/src/go/pt-k8s-debug-collector/dumper/logs.go new file mode 100644 index 000000000..683244066 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/logs.go @@ -0,0 +1,62 @@ +package dumper + +import ( + "context" + "io" + "os" + + corev1 "k8s.io/api/core/v1" +) + +const ( + MaxRamPerLog = 50 * 1024 * 1024 // 50MB RAM limit before spilling to disk +) + +func (d *Dumper) exportPodLogs(ctx context.Context, pod corev1.Pod) error { + containers := append(pod.Spec.InitContainers, pod.Spec.Containers...) + + for _, c := range containers { + logOptions := &corev1.PodLogOptions{Container: c.Name} + req := d.clientSet.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, logOptions) + + stream, err := req.Stream(ctx) + if err != nil { + return err + } + + tmp, err := os.CreateTemp("", "pod-log-*.log") + if err != nil { + stream.Close() + return err + } + defer func() { + tmp.Close() + os.Remove(tmp.Name()) + }() + + _, err = io.Copy(tmp, stream) + stream.Close() + if err != nil { + return err + } + + info, err := tmp.Stat() + if err != nil { + return err + } + + if _, err := tmp.Seek(0, 0); err != nil { + return err + } + + path := d.PodLogPath(pod.Namespace, pod.Name, c.Name) + if err := d.archive.WriteFile(path, tmp, info.Size()); err != nil { + return err + } + + tmp.Close() + os.Remove(tmp.Name()) + } + + return nil +} diff --git a/src/go/pt-k8s-debug-collector/dumper/paths.go b/src/go/pt-k8s-debug-collector/dumper/paths.go new file mode 100644 index 000000000..cac5d4d6a --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/paths.go @@ -0,0 +1,41 @@ +package dumper + +import "path/filepath" + +// ////summary.txt +func (d *Dumper) PodSummaryPath(namespace, podName string) string { + return filepath.Join(d.location, namespace, podName, "summary.txt") +} + +// //// +func (d *Dumper) PodIndividualFilesPath(namespace, podName, internalFilePath string) string { + return filepath.Join(d.location, namespace, podName, internalFilePath) +} + +// ///secrets/.yaml +func (d *Dumper) PodSecretsPath(namespace, secretName string) string { + return filepath.Join(d.location, namespace, secretName+".yaml") +} + +// ///secrets/ +func (d *Dumper) PodRawSecretsPath(namespace, secretName string) string { + return filepath.Join(d.location, namespace, secretName) +} + +// //.log +func (d *Dumper) DumperLogPath(logPrefix string) string { + return filepath.Join(d.location, logPrefix+".log") +} + +// ///.yaml +func (d *Dumper) PodResourcePath(namespace, resourceName string) string { + if namespace == "" { + namespace = "cluster-scope" + } + return filepath.Join(d.location, namespace, resourceName+".yaml") +} + +// ////.log +func (d *Dumper) PodLogPath(namespace, podName, containerName string) string { + return filepath.Join(d.location, namespace, podName, containerName+".log") +} diff --git a/src/go/pt-k8s-debug-collector/dumper/resources.go b/src/go/pt-k8s-debug-collector/dumper/resources.go new file mode 100644 index 000000000..c3b70b143 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/resources.go @@ -0,0 +1,120 @@ +package dumper + +import ( + "fmt" + "log" + "regexp" + "slices" + "strings" +) + +var resourcesRe = regexp.MustCompile(`(\w+\.(\w+).percona\.com)`) + +func (d *Dumper) addPg1() error { + return nil +} + +func (d *Dumper) addPg2() error { + return nil +} + +func (d *Dumper) addPxc() error { + filepaths := []string{ + "var/lib/mysql/mysqld-error.log", + "var/lib/mysql/innobackup.backup.log", + "var/lib/mysql/innobackup.move.log", + "var/lib/mysql/innobackup.prepare.log", + "var/lib/mysql/grastate.dat", + "var/lib/mysql/gvwstate.dat", + "var/lib/mysql/mysqld.post.processing.log", + "var/lib/mysql/auto.cnf", + } + + d.individualFiles = append(d.individualFiles, individualFile{ + resourceName: "pxc", + containerName: "logs", + filepaths: filepaths, + }) + return nil +} + +func (d *Dumper) addPs() error { + return nil +} + +func (d *Dumper) addPsmdb() error { + return nil +} + +// TODO: review and make simplier +func (d *Dumper) autoCustomResource() ([]string, error) { + apiGroupList, err := d.clientSet.DiscoveryClient.ServerGroups() + if err != nil { + return nil, fmt.Errorf("error getting server groups: %w", err) + } + var resourceNames string + var beforeSorting []string + uniqueResourceNames := make(map[string]bool) + for _, group := range apiGroupList.Groups { + for _, version := range group.Versions { + resourceList, err := d.clientSet.DiscoveryClient.ServerResourcesForGroupVersion(version.GroupVersion) + if err != nil { + log.Printf("Warning: Could not get resources for GroupVersion %s: %v", version.GroupVersion, err) + continue + } + for _, resource := range resourceList.APIResources { + if resource.Name != "" && !strings.Contains(resource.Name, "/") { + if group.Name != "" { + uniqueResourceNames[resource.Name+"."+group.Name] = true + } else { + uniqueResourceNames[resource.Name] = true + } + } + } + } + } + for name := range uniqueResourceNames { + beforeSorting = append(beforeSorting, name) + } + slices.Sort(beforeSorting) + for _, name := range beforeSorting { + resourceNames = resourceNames + name + "\n" + } + + matches := resourcesRe.FindAllStringSubmatch(resourceNames, -1) + if len(matches) == 0 { + return []string{"none"}, nil + } + + uniqueResources := make(map[string]bool, 0) + for _, match := range matches { + uniqueResources[match[2]] = true + } + + result := make([]string, 0) + for res := range uniqueResources { + result = append(result, res) + } + + return result, nil +} + +// TODO: rewrite to use map or switch +func resourceType(s string) string { + if s == "auto" { + return "auto" + } else if s == "pxc" || strings.HasPrefix(s, "pxc/") { + return "pxc" + } else if s == "psmdb" || strings.HasPrefix(s, "psmdb/") { + return "psmdb" + } else if s == "pg" || strings.HasPrefix(s, "pg/") { + return "pg" + } else if s == "pgo" || strings.HasPrefix(s, "pgo/") { + return "pg" + } else if s == "pgv2" || strings.HasPrefix(s, "pgv2/") { + return "pgv2" + } else if s == "ps" || strings.HasPrefix(s, "ps/") { + return "ps" + } + return s +} diff --git a/src/go/pt-k8s-debug-collector/dumper/secrets.go b/src/go/pt-k8s-debug-collector/dumper/secrets.go new file mode 100644 index 000000000..c851fb9e4 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/secrets.go @@ -0,0 +1,126 @@ +package dumper + +import ( + "bytes" + "context" + "fmt" + "log" + "os/exec" + "strings" + + "go.yaml.in/yaml/v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +var ( + TargetSSLKeys = []string{"tls.crt", "ca.crt", "tls-ca.crt"} +) + +func (d *Dumper) dumpSecrets(ctx context.Context, namespace string) error { + secretList, err := d.clientSet.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to get secrets: %w", err) + } + + for _, secret := range secretList.Items { + secretName := secret.GetName() + if secret.Type == corev1.SecretTypeTLS || secret.Type == corev1.SecretTypeOpaque { + result, err := decodeCertToBytes(secret) + if err != nil { + log.Printf("Error decoding cert %s: %v", secretName, err) + } + + if len(result) != 0 { + path := d.PodRawSecretsPath(namespace, secretName) + if err := d.archive.WriteVirtualFile(path, result); err != nil { + return err + } + } + } + + itemMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&secret) + if err != nil { + return fmt.Errorf("error converting secret %s: %w", secretName, err) + } + + if strings.Contains(secretName, "pgbouncer") { + itemMap["data"] = map[string]interface{}{ + "warning": "pt-k8s-debug-collector is not collecting secret details of pgbouncer", + } + } + + // TODO: CHECK WHAT FIELDS SHOULD BE DELETED + if meta, ok := itemMap["metadata"].(map[string]interface{}); ok { + delete(meta, "managedFields") + delete(meta, "resourceVersion") + delete(meta, "uid") + delete(meta, "selfLink") + } + + yamlBytes, err := yaml.Marshal(itemMap) + if err != nil { + return fmt.Errorf("failed to marshal secret: %w", err) + } + path := d.PodSecretsPath(namespace, secretName) + if err := d.archive.WriteVirtualFile(path, yamlBytes); err != nil { + return err + } + } + return nil +} + +// TODO: maybe replace "openssl -text" with crypto/x509 + struct dump to json +func decodeCertToBytes(secret corev1.Secret) ([]byte, error) { + var result []byte + + for _, key := range TargetSSLKeys { + certData, ok := secret.Data[key] + if !ok { + continue + } + + header := fmt.Sprintf("\n--- Decoded %s ---\n", key) + result = append(result, []byte(header)...) + cmd := exec.Command("openssl", "x509", "-noout", "-text") + cmd.Stdin = bytes.NewReader(certData) + + var outb, errb bytes.Buffer + cmd.Stdout = &outb + cmd.Stderr = &errb + + err := cmd.Run() + if err != nil { + errMsg := fmt.Sprintf("ERROR running openssl on key %s: %v\nStderr: %s\n", key, err, errb.String()) + result = append(result, []byte(errMsg)...) + } else { + result = append(result, outb.Bytes()...) + } + } + + return result, nil +} + +func (d *Dumper) getSecretValueFromPod(ctx context.Context, pod corev1.Pod, secretName string) (string, error) { + for _, volume := range pod.Spec.Volumes { + if volume.Secret == nil { + continue + } + + secret, err := d.clientSet.CoreV1().Secrets(pod.Namespace).Get(ctx, volume.Secret.SecretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return "", fmt.Errorf("error fetching secret '%s/%s': %w", pod.Namespace, volume.Secret.SecretName, err) + } + + secretValueBytes, found := secret.Data[secretName] + if found { + return string(secretValueBytes), nil + } + } + return "", fmt.Errorf("could not find any secret with name %s in Pod '%s/%s", secretName, pod.Namespace, pod.Name) +} diff --git a/src/go/pt-k8s-debug-collector/dumper/summary.go b/src/go/pt-k8s-debug-collector/dumper/summary.go new file mode 100644 index 000000000..4e05acb89 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/summary.go @@ -0,0 +1,121 @@ +package dumper + +import ( + "bytes" + "context" + "fmt" + "log" + "net/http" + "os/exec" + + corev1 "k8s.io/api/core/v1" +) + +func (d *Dumper) getSummary(ctx context.Context, job exportJob, crType string, location string) { + if !d.skipPodSummary { + output, err := d.getPodSummary(ctx, job.Pod, crType) + if err != nil { + log.Printf("error while creating summary for %q pod and %q namespace: %s", job.Pod.Name, job.Pod.Namespace, err) + err = d.archive.WriteVirtualFile(location, []byte(err.Error())) + if err != nil { + log.Printf("Error: create summary errors archive for pod %s in namespace %s: %v", job.Pod.Name, job.Pod.Namespace, err) + } + } else { + log.Printf("Created summary for pod/namespace %q/%q, Writing to dump", job.Pod.Name, job.Pod.Namespace) + err = d.archive.WriteVirtualFile(location, output) + if err != nil { + log.Printf("error while writing summary for %q pod and %q namespace to dump: %s", job.Pod.Name, job.Pod.Namespace, err) + } + } + } +} + +func (d *Dumper) getPodSummary(ctx context.Context, pod corev1.Pod, crName string) ([]byte, error) { + var ( + summCmdName string + ports string + summCmdArgs []string + ) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + switch crName { + case "pxc", "ps": + var port string + if d.forwardport != "" { + port = d.forwardport + } else { + port = "" + } + + pass, err := d.getSecretValueFromPod(ctx, pod, "root") + if err != nil { + return nil, fmt.Errorf("failed to get password from pxc/ps users secret: %w", err) + } + + ports = port + ":3306" + localport, err := d.portForwardPod(ctx, pod, ports) + if err != nil { + return nil, err + } + + summCmdName = "pt-mysql-summary" + summCmdArgs = []string{"--host=127.0.0.1", "--port=" + fmt.Sprintf("%d", localport), "--user=root", "--password=" + pass} + + case "pgv2", "pg": + scriptURL := "https://raw.githubusercontent.com/percona/support-snippets/master/postgresql/pg_gather/gather.sql" + resp, err := http.Get(scriptURL) + if err != nil { + return nil, fmt.Errorf("error fetching SQL script: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to fetch SQL script, status code: %d", resp.StatusCode) + } + command := []string{"psql", "-X", "-f", "-"} + + outb, errb, err := d.executeInPod(ctx, command, pod, "database", resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to execute command inside pod stdout: %s\n, stderr: %s: %w", outb.String(), errb.String(), err) + } + return outb.Bytes(), nil + + case "psmdb": + var port string + if d.forwardport != "" { + port = d.forwardport + } else { + port = "" + } + + user, err := d.getSecretValueFromPod(ctx, pod, "MONGODB_DATABASE_ADMIN_USER") + if err != nil { + return nil, fmt.Errorf("get user name from psmdb users secret: %w", err) + } + pass, err := d.getSecretValueFromPod(ctx, pod, "MONGODB_DATABASE_ADMIN_PASSWORD") + if err != nil { + return nil, fmt.Errorf("get password from psmdb users secret: %w", err) + } + + ports = port + ":27017" + localport, err := d.portForwardPod(ctx, pod, ports) + if err != nil { + return nil, err + } + + summCmdName = "pt-mongodb-summary" + summCmdArgs = []string{"--username=" + user, "--password=" + string(pass), "--authenticationDatabase=admin", "127.0.0.1:" + fmt.Sprintf("%d", localport)} + } + + var outb, errb bytes.Buffer + cmd := exec.Command(summCmdName, summCmdArgs...) + cmd.Stdout = &outb + cmd.Stderr = &errb + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("stderr: %s\nstdout: %s \nerr: %w", errb.String(), outb.String(), err) + } + return outb.Bytes(), nil +} diff --git a/src/go/pt-k8s-debug-collector/dumper/tar.go b/src/go/pt-k8s-debug-collector/dumper/tar.go new file mode 100644 index 000000000..b4a902783 --- /dev/null +++ b/src/go/pt-k8s-debug-collector/dumper/tar.go @@ -0,0 +1,66 @@ +package dumper + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "sync" + "time" +) + +// tarWriter is a thread-safe wrapper around tar.Writer +type tarWriter struct { + mu sync.Mutex + tw *tar.Writer + gz *gzip.Writer + out *os.File +} + +func NewTarWriter(filename string) (*tarWriter, error) { + f, err := os.Create(filename) + if err != nil { + return nil, err + } + gz := gzip.NewWriter(f) + + return &tarWriter{ + tw: tar.NewWriter(gz), + gz: gz, + out: f, + }, nil +} + +func (s *tarWriter) Close() { + s.tw.Close() + s.gz.Close() + s.out.Close() +} + +func (t *tarWriter) WriteFile(path string, r io.Reader, size int64) error { + t.mu.Lock() + defer t.mu.Unlock() + + hdr := &tar.Header{ + Name: path, + Size: size, + Mode: 0644, + ModTime: time.Now(), + } + + if err := t.tw.WriteHeader(hdr); err != nil { + return err + } + + _, err := io.CopyN(t.tw, r, size) + return err +} + +func (t *tarWriter) WriteVirtualFile(path string, content []byte) error { + return t.WriteFile( + path, + bytes.NewReader(content), + int64(len(content)), + ) +} diff --git a/src/go/pt-k8s-debug-collector/main.go b/src/go/pt-k8s-debug-collector/main.go index 71d95dae2..3d2301003 100644 --- a/src/go/pt-k8s-debug-collector/main.go +++ b/src/go/pt-k8s-debug-collector/main.go @@ -52,10 +52,14 @@ func main() { resource += "/" + clusterName } - d := dumper.New("", namespace, resource, kubeconfig, forwardport, skipPodSummary) + d, err := dumper.New("cluster-dump", namespace, kubeconfig, forwardport, resource, skipPodSummary) + if err != nil { + log.Println("Error:", err) + os.Exit(1) + } log.Println("Start collecting cluster data") - err := d.DumpCluster() + err = d.DumpCluster() if err != nil { log.Println("Error:", err) os.Exit(1) diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index a64e5fddf..77a46f562 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -2,14 +2,16 @@ package main import ( "bytes" + "log" "os" "os/exec" "path" "regexp" + "slices" "strings" "testing" - "golang.org/x/exp/slices" + "github.com/percona/percona-toolkit/src/go/tests/utils" ) /* @@ -34,20 +36,45 @@ We do not explicitly test --kubeconfig and --forwardport options, because they a /* Tests TODO: - - Test clusters with custom user and secrets. With the way we currently test, we just need to create a cluster with particular options. But it is already time and resource consuming operation. So we need to either test only getCR function or create a mock cluster, or find a better way to deploy test clusters. */ +// You need to have anydbver in path to start tests (https://github.com/ihanick/anydbver) + +func TestMain(m *testing.M) { + //args := []string{"deploy", "k8s-pg:2.8.0", "k8s-pxc:1.18.0","k8s-psmdb:1.21.1", "k8s-ps:1.0.0", } + + // For some reason ps is not reporting correctly that it is ready, + // you can deploy it manually using above command but it will hang until timeout + // even if it's deployed and ready + + // TODO: fix ps and add pgv1 + args := []string{"deploy", "k8s-pg:2.8.0", "k8s-pxc:1.18.0", "k8s-psmdb:1.21.1"} + utils.DeployAnyDbVer(args) + + exitCode := m.Run() + if exitCode == 0 { + log.Println("Tests finished succesfully, destroying deployments") + // Comment this if you don't want to destroy deployments after tests + err := utils.CleanUpAnyDbVer() + if err != nil { + log.Fatalf("there was an error when destroying deloyments: %v", err) + } + } + os.Exit(exitCode) +} + /* Tests collection of the individual files by pt-k8s-debug-collector. Requires running K8SPXC instance and kubectl, configured to access that instance by default. */ func TestIndividualFiles(t *testing.T) { - if os.Getenv("KUBECONFIG_PXC") == "" { - t.Skip("TestIndividualFiles requires K8SPXC") + config, err := utils.GetKubeConfigString() + if err != nil { + t.Fatalf("error getting config for kube: %v", err) } tests := []struct { name string @@ -91,7 +118,7 @@ func TestIndividualFiles(t *testing.T) { } for _, resource := range []string{"pxc", "auto"} { - cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", os.Getenv("KUBECONFIG_PXC"), "--forwardport", os.Getenv("FORWARDPORT"), "--resource", resource) + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } @@ -118,6 +145,10 @@ func TestIndividualFiles(t *testing.T) { Tests for supported values of the --resource option */ func TestResourceOption(t *testing.T) { + config, err := utils.GetKubeConfigString() + if err != nil { + t.Fatalf("error getting config for kube: %v", err) + } testcmd := []string{"sh", "-c", "tar -tf cluster-dump.tar.gz --wildcards '*/summary.txt' 2>/dev/null | wc -l"} tests := []struct { name string @@ -194,16 +225,18 @@ func TestResourceOption(t *testing.T) { } for _, test := range tests { - cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", test.kubeconfig, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource) + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } + defer func() { cmd = exec.Command("rm", "-f", "cluster-dump.tar.gz") if err := cmd.Run(); err != nil { t.Errorf("error cleaning up test data: %s", err.Error()) } }() + out, err := exec.Command(testcmd[0], testcmd[1:]...).Output() if err != nil { t.Errorf("test %s, error running command %s:\n%s\n\nCommand output:\n%s", test.name, testcmd, err.Error(), out) @@ -218,6 +251,10 @@ func TestResourceOption(t *testing.T) { PT-2299 - collect openssl x509 certificate information for each secret */ func TestSSLResourceOption(t *testing.T) { + config, err := utils.GetKubeConfigString() + if err != nil { + t.Fatalf("error getting config for kube: %v", err) + } tests := []struct { name string resource string @@ -343,7 +380,7 @@ func TestSSLResourceOption(t *testing.T) { } for _, test := range tests { - cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", test.kubeconfig, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource) + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } @@ -369,6 +406,10 @@ func TestSSLResourceOption(t *testing.T) { Tests for option --skip-pod-summary */ func TestPT_2453(t *testing.T) { + config, err := utils.GetKubeConfigString() + if err != nil { + t.Fatalf("error getting config for kube: %v", err) + } testcmd := []string{"sh", "-c", "tar -tf cluster-dump.tar.gz --wildcards '*/summary.txt' 2>/dev/null | wc -l"} tests := []struct { name string @@ -445,7 +486,7 @@ func TestPT_2453(t *testing.T) { } for _, test := range tests { - cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", test.kubeconfig, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource, "--skip-pod-summary") + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource, "--skip-pod-summary") if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s\nCommand: %s", err.Error(), cmd.String()) } @@ -484,6 +525,10 @@ func TestVersionOption(t *testing.T) { If we handle error properly */ func TestPT_2169(t *testing.T) { + config, err := utils.GetKubeConfigString() + if err != nil { + t.Fatalf("error getting config for kube: %v", err) + } busyport, _ := os.Getwd() // we are using wrong socket for ssh tunnel here to ensure we get error testcmd := []string{"sh", "-c", "tar -xf cluster-dump.tar.gz --wildcards '*/summary.txt' --to-command 'grep stderr:' 2>/dev/null | wc -l"} @@ -511,7 +556,7 @@ func TestPT_2169(t *testing.T) { } for _, test := range tests { - cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", test.kubeconfig, "--forwardport", test.port, "--resource", test.resource) + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", test.port, "--resource", test.resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } diff --git a/src/go/tests/utils/anydbver.go b/src/go/tests/utils/anydbver.go new file mode 100644 index 000000000..605f3307c --- /dev/null +++ b/src/go/tests/utils/anydbver.go @@ -0,0 +1,51 @@ +package utils + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func RunAnyDbVer(args ...string) (string, error) { + cmd := exec.Command("anydbver", args...) + output, err := cmd.CombinedOutput() + outputStr := strings.TrimSpace(string(output)) + if err != nil { + return outputStr, fmt.Errorf("failed to run anydbver: %s \nOutput: %s", err, outputStr) + } + + return outputStr, nil +} + +func DeployAnyDbVer(args []string) { + log.Printf("Starting deployment") + output, err := RunAnyDbVer(args...) + if err != nil { + log.Printf("Fail when deploying: %v \nOutput: %s\n", err, output) + return + } + log.Printf("Successfully deployed \nOutput: %s\n", output) +} + +func GetKubeConfigString() (string, error) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to get user home directory: %w", err) + } + kubeconfig = filepath.Join(home, ".kube", "config") + } + return kubeconfig, nil +} + +func CleanUpAnyDbVer() error { + _, err := RunAnyDbVer("destroy") + if err != nil { + return fmt.Errorf("cleanup failed: %w", err) + } + return nil +} From f20025054333e9cfbf9d4ed1ad110179fd3a415f Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 8 Jan 2026 23:35:12 +0200 Subject: [PATCH 04/21] fixes --- .../pt-k8s-debug-collector/dumper/dumper.go | 2 ++ .../dumper/kube_utils.go | 31 +++++++++++++------ .../pt-k8s-debug-collector/dumper/summary.go | 12 +++---- src/go/pt-k8s-debug-collector/main_test.go | 9 +----- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index 1cbe851aa..12aa2d5d3 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -37,6 +37,7 @@ type Dumper struct { mode int64 crTypes []string forwardport string + usedPorts sync.Map skipPodSummary bool sslSecrets map[string]bool @@ -104,6 +105,7 @@ func New(location, namespace, kubeconfig, forwardport, resource string, skipPodS discoveryClient: discclient, restConfig: config, logger: NewSafeLogger(), + usedPorts: sync.Map{}, } if resource == "none" || resource == "" { diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index f7c12ebd9..2aac7f036 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -3,13 +3,14 @@ package dumper import ( "bytes" "context" + "errors" "fmt" "io" "log" "net/http" "net/url" "path" - "strings" + "strconv" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" @@ -18,10 +19,27 @@ import ( "k8s.io/client-go/transport/spdy" ) +var ( + ERR_PORT_ALREADY_FORWARDED = errors.New("this localPort:remotePort is already forwarded") + ERR_LOCAL_PORT_IN_USE = errors.New("this localPort is already in use") +) + /* Forwards ports to a specific pod. Close the returned channel to stop forwarding. */ -func (d *Dumper) portForwardPod(ctx context.Context, pod corev1.Pod, remotePort string) (int, error) { +func (d *Dumper) portForwardPod(ctx context.Context, pod corev1.Pod, localPort string, remotePort string) (int, error) { + localPortParsed, err := strconv.ParseInt(localPort, 10, 32) + if err != nil { + return 0, err + } + + gotRemotePort, loaded := d.usedPorts.LoadOrStore(localPortParsed, remotePort) + if loaded && gotRemotePort == remotePort { + return int(localPortParsed), ERR_PORT_ALREADY_FORWARDED + } else if loaded && gotRemotePort != remotePort { + return 0, ERR_LOCAL_PORT_IN_USE + } + apiURL, err := url.Parse(d.restConfig.Host) if err != nil { return 0, fmt.Errorf("failed to parse config host URL: %w", err) @@ -42,15 +60,8 @@ func (d *Dumper) portForwardPod(ctx context.Context, pod corev1.Pod, remotePort readyChan := make(chan struct{}, 1) out, errOut := new(bytes.Buffer), new(bytes.Buffer) - var ports []string - if strings.Contains(remotePort, ":") { - ports = []string{remotePort} - } else { - ports = []string{fmt.Sprintf(":%s", remotePort)} - } - forwarderCtx, forwarderClose := context.WithCancel(ctx) - forwarder, err := portforward.New(dialer, ports, forwarderCtx.Done(), readyChan, out, errOut) + forwarder, err := portforward.New(dialer, []string{localPort + ":" + remotePort}, forwarderCtx.Done(), readyChan, out, errOut) if err != nil { forwarderClose() return 0, fmt.Errorf("failed to create port forwarder: %w", err) diff --git a/src/go/pt-k8s-debug-collector/dumper/summary.go b/src/go/pt-k8s-debug-collector/dumper/summary.go index 4e05acb89..683873ed6 100644 --- a/src/go/pt-k8s-debug-collector/dumper/summary.go +++ b/src/go/pt-k8s-debug-collector/dumper/summary.go @@ -3,6 +3,7 @@ package dumper import ( "bytes" "context" + "errors" "fmt" "log" "net/http" @@ -33,7 +34,6 @@ func (d *Dumper) getSummary(ctx context.Context, job exportJob, crType string, l func (d *Dumper) getPodSummary(ctx context.Context, pod corev1.Pod, crName string) ([]byte, error) { var ( summCmdName string - ports string summCmdArgs []string ) @@ -54,10 +54,11 @@ func (d *Dumper) getPodSummary(ctx context.Context, pod corev1.Pod, crName strin return nil, fmt.Errorf("failed to get password from pxc/ps users secret: %w", err) } - ports = port + ":3306" - localport, err := d.portForwardPod(ctx, pod, ports) + localport, err := d.portForwardPod(ctx, pod, port, "3306") if err != nil { - return nil, err + if !errors.Is(err, ERR_PORT_ALREADY_FORWARDED) { + return nil, err + } } summCmdName = "pt-mysql-summary" @@ -99,8 +100,7 @@ func (d *Dumper) getPodSummary(ctx context.Context, pod corev1.Pod, crName strin return nil, fmt.Errorf("get password from psmdb users secret: %w", err) } - ports = port + ":27017" - localport, err := d.portForwardPod(ctx, pod, ports) + localport, err := d.portForwardPod(ctx, pod, port, "27017") if err != nil { return nil, err } diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 77a46f562..4b9c48d2a 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -45,14 +45,7 @@ Tests TODO: // You need to have anydbver in path to start tests (https://github.com/ihanick/anydbver) func TestMain(m *testing.M) { - //args := []string{"deploy", "k8s-pg:2.8.0", "k8s-pxc:1.18.0","k8s-psmdb:1.21.1", "k8s-ps:1.0.0", } - - // For some reason ps is not reporting correctly that it is ready, - // you can deploy it manually using above command but it will hang until timeout - // even if it's deployed and ready - - // TODO: fix ps and add pgv1 - args := []string{"deploy", "k8s-pg:2.8.0", "k8s-pxc:1.18.0", "k8s-psmdb:1.21.1"} + args := []string{"deploy", "k8s-pg:2.8.0", "k8s-pxc:1.18.0", "k8s-psmdb:1.21.1", "k8s-ps:1.6.0"} utils.DeployAnyDbVer(args) exitCode := m.Run() From 07f2f06519393e86df53d8ebb3273128d7efd17f Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 9 Jan 2026 19:56:10 +0200 Subject: [PATCH 05/21] fix tests --- .../dumper/kube_utils.go | 4 + src/go/pt-k8s-debug-collector/main_test.go | 333 ++++++++++-------- src/go/tests/utils/anydbver.go | 34 +- src/go/tests/utils/cluster.go | 100 ++++++ 4 files changed, 321 insertions(+), 150 deletions(-) create mode 100644 src/go/tests/utils/cluster.go diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index 2aac7f036..2ee5a235c 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -28,6 +28,10 @@ var ( Forwards ports to a specific pod. Close the returned channel to stop forwarding. */ func (d *Dumper) portForwardPod(ctx context.Context, pod corev1.Pod, localPort string, remotePort string) (int, error) { + if localPort == "" { + localPort = remotePort + } + localPortParsed, err := strconv.ParseInt(localPort, 10, 32) if err != nil { return 0, err diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 4b9c48d2a..35b0275e1 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "log" "os" "os/exec" @@ -10,8 +11,11 @@ import ( "slices" "strings" "testing" + "time" "github.com/percona/percona-toolkit/src/go/tests/utils" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) /* @@ -22,8 +26,27 @@ This test requires: -- KUBECONFIG_PS for K8SPS -- KUBECONFIG_PSMDB for K8SPSMDB -- KUBECONFIG_PG for K8SPG --- KUBECONFIG_PG2 for K8SPG version 2 + -- KUBECONFIG_PG2 for K8SPG version 2 +*/ +var ( + namespaces = []string{ + // "pxc", "ps", "psmdb", "pg", "pgv2", + "pgv2", + } + + resources = []string{ + // "pxc", "ps", "psmdb", "pg", "pgv2", + "pgv2", "auto", "none", + } + + deployments = []string{ + //"k8s-pxc:1.18.0", "k8s-ps:1.0.0", "k8s-psmdb:1.21.1", "k8s-pg:1.6.0", "k8s-pg:2.8.0", + "k8s-pg:2.8.0", + } +) + +/* You can additionally set option FORWARDPORT if you want to use custom port when testing summaries. pt-mysql-summary, mysql, psql, and pt-mongodb-summary must be in the PATH. @@ -44,15 +67,56 @@ Tests TODO: // You need to have anydbver in path to start tests (https://github.com/ihanick/anydbver) +func getKubeClient(kubeconfigPath string) (kubernetes.Interface, error) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + return nil, err + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + func TestMain(m *testing.M) { - args := []string{"deploy", "k8s-pg:2.8.0", "k8s-pxc:1.18.0", "k8s-psmdb:1.21.1", "k8s-ps:1.6.0"} - utils.DeployAnyDbVer(args) + ctx := context.Background() + log.Println("START") + args := []string{"deploy"} + args = append(args, deployments...) + utils.DeployAnyDbVer(ctx, args) + + config, err := utils.GetKubeConfigPath() + if err != nil { + log.Fatalf("could not get a kubeconfig: %s", err) + } + + kubeClient, err := getKubeClient(config) + if err != nil { + log.Fatalf("could not get a kubeclient: %s", err) + } + + for _, ns := range namespaces { + cctx, _ := context.WithTimeout(ctx, time.Minute*10) + + err = utils.WaitForAllStatefulSetReady(cctx, kubeClient, ns) + if err != nil { + log.Fatalf("waiting for all statefullsets to be ready is falied with err: %s", err) + } + + err = utils.WaitForAllPodsReady(cctx, kubeClient, ns) + if err != nil { + log.Fatalf("waiting for all pods to be ready is falied with err: %s", err) + } + } exitCode := m.Run() if exitCode == 0 { log.Println("Tests finished succesfully, destroying deployments") // Comment this if you don't want to destroy deployments after tests - err := utils.CleanUpAnyDbVer() + err := utils.CleanUpAnyDbVer(ctx) if err != nil { log.Fatalf("there was an error when destroying deloyments: %v", err) } @@ -64,18 +128,20 @@ func TestMain(m *testing.M) { Tests collection of the individual files by pt-k8s-debug-collector. Requires running K8SPXC instance and kubectl, configured to access that instance by default. */ -func TestIndividualFiles(t *testing.T) { - config, err := utils.GetKubeConfigString() +func _TestIndividualFiles(t *testing.T) { + config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } tests := []struct { + deployment string name string cmd []string want []string preprocessor func(string) string }{ { + deployment: "pxc", // If the tool collects required log files name: "pxc_logs_list", // tar -tf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/*' @@ -95,6 +161,7 @@ func TestIndividualFiles(t *testing.T) { }, }, { + deployment: "pxc", // If MySQL error log is not empty name: "pxc_mysqld_error_log", // tar --to-command="grep -m 1 -o Version:" -xzf cluster-dump-test.tar.gz --wildcards 'cluster-dump/*/var/lib/mysql/mysqld-error.log' @@ -110,7 +177,12 @@ func TestIndividualFiles(t *testing.T) { }, } - for _, resource := range []string{"pxc", "auto"} { + requestedClusterReports := make(map[string]struct{}, 0) + for _, test := range tests { + requestedClusterReports[test.deployment] = struct{}{} + } + + for resource := range requestedClusterReports { cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) @@ -137,8 +209,8 @@ func TestIndividualFiles(t *testing.T) { /* Tests for supported values of the --resource option */ -func TestResourceOption(t *testing.T) { - config, err := utils.GetKubeConfigString() +func _TestResourceOption(t *testing.T) { + config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } @@ -150,74 +222,67 @@ func TestResourceOption(t *testing.T) { kubeconfig string }{ { - name: "none", - resource: "none", - want: "0", - kubeconfig: "", + name: "none", + resource: "none", + want: "0", }, { - name: "pxc", - resource: "pxc", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PXC"), + name: "pxc", + resource: "pxc", + want: "3", }, { - name: "ps", - resource: "ps", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PS"), + name: "ps", + resource: "ps", + want: "3", }, { - name: "psmdb", - resource: "psmdb", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PSMDB"), + name: "psmdb", + resource: "psmdb", + want: "3", }, { - name: "pg", - resource: "pg", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PG"), + name: "pg", + resource: "pg", + want: "3", }, { - name: "pgv2", - resource: "pgv2", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PG2"), + name: "pgv2", + resource: "pgv2", + want: "3", }, { - name: "auto pxc", - resource: "auto", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PXC"), + name: "auto pxc", + resource: "auto", + want: "3", }, { - name: "auto ps", - resource: "auto", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PS"), + name: "auto ps", + resource: "auto", + want: "3", }, { - name: "auto psmdb", - resource: "auto", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PSMDB"), + name: "auto psmdb", + resource: "auto", + want: "3", }, { - name: "auto pg", - resource: "auto", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PG"), + name: "auto pg", + resource: "auto", + want: "3", }, { - name: "auto pgv2", - resource: "auto", - want: "3", - kubeconfig: os.Getenv("KUBECONFIG_PG2"), + name: "auto pgv2", + resource: "auto", + want: "3", }, } for _, test := range tests { + if !slices.Contains(resources, test.resource) { + continue + } + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) @@ -244,16 +309,15 @@ func TestResourceOption(t *testing.T) { PT-2299 - collect openssl x509 certificate information for each secret */ func TestSSLResourceOption(t *testing.T) { - config, err := utils.GetKubeConfigString() + config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } tests := []struct { - name string - resource string - cmds [][]string // slice of commands to execute - want []string // slice of expected results - kubeconfig string + name string + resource string + cmds [][]string // slice of commands to execute + want []string // slice of expected results }{ { name: "auto pxc", @@ -280,7 +344,6 @@ func TestSSLResourceOption(t *testing.T) { "Certificate", "tls.crt", }, - kubeconfig: os.Getenv("KUBECONFIG_PXC"), }, { name: "auto ps", @@ -301,7 +364,6 @@ func TestSSLResourceOption(t *testing.T) { "Certificate", "tls.crt", }, - kubeconfig: os.Getenv("KUBECONFIG_PS"), }, { name: "auto psmdb", @@ -328,7 +390,6 @@ func TestSSLResourceOption(t *testing.T) { "Certificate", "tls.crt", }, - kubeconfig: os.Getenv("KUBECONFIG_PSMDB"), }, { name: "auto pg", @@ -349,7 +410,6 @@ func TestSSLResourceOption(t *testing.T) { "tls.crt", "Certificate", }, - kubeconfig: os.Getenv("KUBECONFIG_PG"), }, { name: "auto pgv2", @@ -368,28 +428,30 @@ func TestSSLResourceOption(t *testing.T) { "root.crt", "Certificate", }, - kubeconfig: os.Getenv("KUBECONFIG_PG2"), }, } for _, test := range tests { + if !slices.Contains(resources, test.resource) { + continue + } cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } - defer func() { - cmd = exec.Command("rm", "-f", "cluster-dump.tar.gz") - if err := cmd.Run(); err != nil { - t.Errorf("error cleaning up test data: %s", err.Error()) - } - }() + // defer func() { + // cmd = exec.Command("rm", "-f", "cluster-dump.tar.gz") + // if err := cmd.Run(); err != nil { + // t.Errorf("error cleaning up test data: %s", err.Error()) + // } + // }() for ind, testcmd := range test.cmds { out, err := exec.Command(testcmd[0], testcmd[1:]...).Output() if err != nil { t.Errorf("test %s, error running command %s:\n%s\n\nCommand output:\n%s", test.name, testcmd, err.Error(), out) } if strings.TrimRight(bytes.NewBuffer(out).String(), "\n") != test.want[ind] { - t.Errorf("test %s, output is not as expected\nOutput: %s\nWanted: %s", test.name, out, test.want) + t.Errorf("test %s, output is not as expected\nOutput: %s\nWanted: %s", test.name, out, test.want[ind]) } } } @@ -398,87 +460,79 @@ func TestSSLResourceOption(t *testing.T) { /* Tests for option --skip-pod-summary */ -func TestPT_2453(t *testing.T) { - config, err := utils.GetKubeConfigString() +func _TestPT_2453(t *testing.T) { + config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } testcmd := []string{"sh", "-c", "tar -tf cluster-dump.tar.gz --wildcards '*/summary.txt' 2>/dev/null | wc -l"} tests := []struct { - name string - resource string - want string - kubeconfig string + name string + resource string + want string }{ { - name: "none", - resource: "none", - want: "0", - kubeconfig: "", + name: "none", + resource: "none", + want: "0", }, { - name: "pxc", - resource: "pxc", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PXC"), + name: "pxc", + resource: "pxc", + want: "0", }, { - name: "ps", - resource: "ps", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PS"), + name: "ps", + resource: "ps", + want: "0", }, { - name: "psmdb", - resource: "psmdb", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PSMDB"), + name: "psmdb", + resource: "psmdb", + want: "0", }, { - name: "pg", - resource: "pg", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PG"), + name: "pg", + resource: "pg", + want: "0", }, { - name: "pgv2", - resource: "pgv2", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PG2"), + name: "pgv2", + resource: "pgv2", + want: "0", }, { - name: "auto pxc", - resource: "auto", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PXC"), + name: "auto pxc", + resource: "auto", + want: "0", }, { - name: "auto ps", - resource: "auto", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PS"), + name: "auto ps", + resource: "auto", + want: "0", }, { - name: "auto psmdb", - resource: "auto", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PSMDB"), + name: "auto psmdb", + resource: "auto", + want: "0", }, { - name: "auto pg", - resource: "auto", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PG"), + name: "auto pg", + resource: "auto", + want: "0", }, { - name: "auto pgv2", - resource: "auto", - want: "0", - kubeconfig: os.Getenv("KUBECONFIG_PG2"), + name: "auto pgv2", + resource: "auto", + want: "0", }, } for _, test := range tests { + if !slices.Contains(resources, test.resource) { + continue + } + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", test.resource, "--skip-pod-summary") if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s\nCommand: %s", err.Error(), cmd.String()) @@ -502,7 +556,7 @@ func TestPT_2453(t *testing.T) { /* Option --version */ -func TestVersionOption(t *testing.T) { +func __TestVersionOption(t *testing.T) { out, err := exec.Command("../../../bin/"+toolname, "--version").Output() if err != nil { t.Errorf("error executing %s --version: %s", toolname, err.Error()) @@ -517,8 +571,8 @@ func TestVersionOption(t *testing.T) { /* If we handle error properly */ -func TestPT_2169(t *testing.T) { - config, err := utils.GetKubeConfigString() +func _TestPT_2169(t *testing.T) { + config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } @@ -526,29 +580,30 @@ func TestPT_2169(t *testing.T) { testcmd := []string{"sh", "-c", "tar -xf cluster-dump.tar.gz --wildcards '*/summary.txt' --to-command 'grep stderr:' 2>/dev/null | wc -l"} tests := []struct { - name string - resource string - want string - port string - kubeconfig string + name string + resource string + want string + port string }{ { - name: "pxc with busy port", - resource: "pxc", - want: "3", - port: busyport, - kubeconfig: os.Getenv("KUBECONFIG_PXC"), + name: "pxc with busy port", + resource: "pxc", + want: "3", + port: busyport, }, { - name: "pg no error", - resource: "pg", - want: "0", - port: os.Getenv("FORWARDPORT"), - kubeconfig: os.Getenv("KUBECONFIG_PG"), + name: "pg no error", + resource: "pg", + want: "0", + port: os.Getenv("FORWARDPORT"), }, } for _, test := range tests { + if !slices.Contains(resources, test.resource) { + continue + } + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", test.port, "--resource", test.resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) diff --git a/src/go/tests/utils/anydbver.go b/src/go/tests/utils/anydbver.go index 605f3307c..f1e68af56 100644 --- a/src/go/tests/utils/anydbver.go +++ b/src/go/tests/utils/anydbver.go @@ -1,7 +1,9 @@ package utils import ( + "context" "fmt" + "io" "log" "os" "os/exec" @@ -9,10 +11,20 @@ import ( "strings" ) -func RunAnyDbVer(args ...string) (string, error) { - cmd := exec.Command("anydbver", args...) - output, err := cmd.CombinedOutput() - outputStr := strings.TrimSpace(string(output)) +func RunAnyDbVer(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "anydbver", args...) + + var outputBuilder strings.Builder + + stdout := io.MultiWriter(&outputBuilder, os.Stdout) + stderr := io.MultiWriter(&outputBuilder, os.Stderr) + + cmd.Stdout = stdout + cmd.Stderr = stderr + + err := cmd.Run() + outputStr := strings.TrimSpace(outputBuilder.String()) + if err != nil { return outputStr, fmt.Errorf("failed to run anydbver: %s \nOutput: %s", err, outputStr) } @@ -20,17 +32,17 @@ func RunAnyDbVer(args ...string) (string, error) { return outputStr, nil } -func DeployAnyDbVer(args []string) { +func DeployAnyDbVer(ctx context.Context, args []string) error { log.Printf("Starting deployment") - output, err := RunAnyDbVer(args...) + output, err := RunAnyDbVer(ctx, args...) if err != nil { - log.Printf("Fail when deploying: %v \nOutput: %s\n", err, output) - return + return fmt.Errorf("Fail when deploying: %v \nOutput: %s\n", err, output) } log.Printf("Successfully deployed \nOutput: %s\n", output) + return nil } -func GetKubeConfigString() (string, error) { +func GetKubeConfigPath() (string, error) { kubeconfig := os.Getenv("KUBECONFIG") if kubeconfig == "" { home, err := os.UserHomeDir() @@ -42,8 +54,8 @@ func GetKubeConfigString() (string, error) { return kubeconfig, nil } -func CleanUpAnyDbVer() error { - _, err := RunAnyDbVer("destroy") +func CleanUpAnyDbVer(ctx context.Context) error { + _, err := RunAnyDbVer(ctx, "destroy") if err != nil { return fmt.Errorf("cleanup failed: %w", err) } diff --git a/src/go/tests/utils/cluster.go b/src/go/tests/utils/cluster.go new file mode 100644 index 000000000..cc4e20e50 --- /dev/null +++ b/src/go/tests/utils/cluster.go @@ -0,0 +1,100 @@ +package utils + +import ( + "context" + "log" + "time" + + "github.com/goforj/godump" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +func WaitForAllStatefulSetReady(ctx context.Context, client kubernetes.Interface, namespace string) error { + return wait.PollUntilContextCancel(ctx, 15*time.Second, true, func(ctx context.Context) (bool, error) { + stsList, err := client.AppsV1().StatefulSets(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + + desiredNum := len(stsList.Items) + + for _, stsItem := range stsList.Items { + log.Printf("Waiting for StatefulSet: %q", stsItem.Name) + + sts, err := client.AppsV1().StatefulSets(namespace).Get(ctx, stsItem.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + desired := int32(1) + if sts.Spec.Replicas != nil { + desired = *sts.Spec.Replicas + } + + ready := sts.Status.ReadyReplicas + + log.Printf("StatefulSet %q: \n%s", stsItem.Name, godump.DumpStr(sts.Status)) + + log.Printf("StatefulSet %q: desired=%d ready=%d\n", stsItem.Name, desired, ready) + + if ready == desired { + desiredNum-- + } + } + + if desiredNum <= 0 { + return true, nil + } + + return false, nil + }) +} + +func WaitForAllPodsReady( + ctx context.Context, + client kubernetes.Interface, + namespace string, +) error { + return wait.PollUntilContextCancel(ctx, 15*time.Second, true, func(ctx context.Context) (bool, error) { + pods, err := client.CoreV1(). + Pods(namespace). + List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + + if len(pods.Items) == 0 { + return false, nil + } + + for i, pod := range pods.Items { + if pod.Status.Phase != corev1.PodSucceeded { + continue + } + + log.Printf("%dPOD:\n%s", i, godump.DumpStr(pod)) + + if pod.Status.Phase != corev1.PodRunning { + return false, nil + } + + if !isPodReady(&pod) { + return false, nil + } + } + + return true, nil + }) +} + +func isPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.ContainerStatuses { + if !cond.Ready { + return false + } + } + return true +} From 4d477c2525ab6e86152324e66b8291ccf700e250 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 9 Jan 2026 20:02:56 +0200 Subject: [PATCH 06/21] fix typo --- src/go/pt-k8s-debug-collector/dumper/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/resources.go b/src/go/pt-k8s-debug-collector/dumper/resources.go index c3b70b143..766a618ef 100644 --- a/src/go/pt-k8s-debug-collector/dumper/resources.go +++ b/src/go/pt-k8s-debug-collector/dumper/resources.go @@ -46,7 +46,7 @@ func (d *Dumper) addPsmdb() error { return nil } -// TODO: review and make simplier +// TODO: review and make simple func (d *Dumper) autoCustomResource() ([]string, error) { apiGroupList, err := d.clientSet.DiscoveryClient.ServerGroups() if err != nil { From 707865b4921688007a4473d3be9a0eefd68f1f8e Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 9 Jan 2026 22:55:00 +0200 Subject: [PATCH 07/21] fix tests --- src/go/pt-k8s-debug-collector/main_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 35b0275e1..485bcda25 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -31,18 +31,15 @@ This test requires: */ var ( namespaces = []string{ - // "pxc", "ps", "psmdb", "pg", "pgv2", - "pgv2", + "pxc", "ps", "psmdb", "pg", "pgv2", } resources = []string{ - // "pxc", "ps", "psmdb", "pg", "pgv2", - "pgv2", "auto", "none", + "pxc", "ps", "psmdb", "pg", "pgv2", } deployments = []string{ - //"k8s-pxc:1.18.0", "k8s-ps:1.0.0", "k8s-psmdb:1.21.1", "k8s-pg:1.6.0", "k8s-pg:2.8.0", - "k8s-pg:2.8.0", + "k8s-pxc:1.18.0", "k8s-ps:1.0.0", "k8s-psmdb:1.21.1", "k8s-pg:1.6.0", "k8s-pg:2.8.0", } ) From 9159ddcbcb5899e0212f86700b30e04536c41c6f Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Sun, 11 Jan 2026 09:17:36 +0200 Subject: [PATCH 08/21] fix tests --- .../dumper/kube_utils.go | 5 +- src/go/pt-k8s-debug-collector/dumper/paths.go | 4 +- .../pt-k8s-debug-collector/dumper/secrets.go | 6 +- .../pt-k8s-debug-collector/dumper/summary.go | 4 +- src/go/pt-k8s-debug-collector/main_test.go | 162 ++++++------------ src/go/tests/utils/cluster.go | 2 +- 6 files changed, 68 insertions(+), 115 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index 2ee5a235c..b1bfa77e5 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -20,6 +20,9 @@ import ( ) var ( + // Ignore this error only in case when all calls to portForwardPod have the same context. + // If it is not the case, igoning this error may result in a portForward closing while other + // entities is in process of using it. ERR_PORT_ALREADY_FORWARDED = errors.New("this localPort:remotePort is already forwarded") ERR_LOCAL_PORT_IN_USE = errors.New("this localPort is already in use") ) @@ -34,7 +37,7 @@ func (d *Dumper) portForwardPod(ctx context.Context, pod corev1.Pod, localPort s localPortParsed, err := strconv.ParseInt(localPort, 10, 32) if err != nil { - return 0, err + return 0, fmt.Errorf("failed to parse port: %s with err: %s", localPort, err) } gotRemotePort, loaded := d.usedPorts.LoadOrStore(localPortParsed, remotePort) diff --git a/src/go/pt-k8s-debug-collector/dumper/paths.go b/src/go/pt-k8s-debug-collector/dumper/paths.go index cac5d4d6a..2e0a7faaf 100644 --- a/src/go/pt-k8s-debug-collector/dumper/paths.go +++ b/src/go/pt-k8s-debug-collector/dumper/paths.go @@ -14,12 +14,12 @@ func (d *Dumper) PodIndividualFilesPath(namespace, podName, internalFilePath str // ///secrets/.yaml func (d *Dumper) PodSecretsPath(namespace, secretName string) string { - return filepath.Join(d.location, namespace, secretName+".yaml") + return filepath.Join(d.location, namespace, "secrets", secretName+".yaml") } // ///secrets/ func (d *Dumper) PodRawSecretsPath(namespace, secretName string) string { - return filepath.Join(d.location, namespace, secretName) + return filepath.Join(d.location, namespace, "secrets", secretName) } // //.log diff --git a/src/go/pt-k8s-debug-collector/dumper/secrets.go b/src/go/pt-k8s-debug-collector/dumper/secrets.go index c851fb9e4..14bf77c92 100644 --- a/src/go/pt-k8s-debug-collector/dumper/secrets.go +++ b/src/go/pt-k8s-debug-collector/dumper/secrets.go @@ -16,7 +16,7 @@ import ( ) var ( - TargetSSLKeys = []string{"tls.crt", "ca.crt", "tls-ca.crt"} + TargetSSLKeys = []string{"tls.crt", "ca.crt", "tls-ca.crt", "root.crt"} ) func (d *Dumper) dumpSecrets(ctx context.Context, namespace string) error { @@ -82,8 +82,6 @@ func decodeCertToBytes(secret corev1.Secret) ([]byte, error) { continue } - header := fmt.Sprintf("\n--- Decoded %s ---\n", key) - result = append(result, []byte(header)...) cmd := exec.Command("openssl", "x509", "-noout", "-text") cmd.Stdin = bytes.NewReader(certData) @@ -96,6 +94,8 @@ func decodeCertToBytes(secret corev1.Secret) ([]byte, error) { errMsg := fmt.Sprintf("ERROR running openssl on key %s: %v\nStderr: %s\n", key, err, errb.String()) result = append(result, []byte(errMsg)...) } else { + header := fmt.Sprintf("\n--- Decoded %s ---\n", key) + result = append(result, []byte(header)...) result = append(result, outb.Bytes()...) } } diff --git a/src/go/pt-k8s-debug-collector/dumper/summary.go b/src/go/pt-k8s-debug-collector/dumper/summary.go index 683873ed6..a95522e70 100644 --- a/src/go/pt-k8s-debug-collector/dumper/summary.go +++ b/src/go/pt-k8s-debug-collector/dumper/summary.go @@ -102,7 +102,9 @@ func (d *Dumper) getPodSummary(ctx context.Context, pod corev1.Pod, crName strin localport, err := d.portForwardPod(ctx, pod, port, "27017") if err != nil { - return nil, err + if !errors.Is(err, ERR_PORT_ALREADY_FORWARDED) { + return nil, err + } } summCmdName = "pt-mongodb-summary" diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 485bcda25..e19f3ea23 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "fmt" "log" "os" "os/exec" @@ -35,7 +36,7 @@ var ( } resources = []string{ - "pxc", "ps", "psmdb", "pg", "pgv2", + "pxc", "ps", "psmdb", "pg", "pgv2", "auto", "none", } deployments = []string{ @@ -125,7 +126,7 @@ func TestMain(m *testing.M) { Tests collection of the individual files by pt-k8s-debug-collector. Requires running K8SPXC instance and kubectl, configured to access that instance by default. */ -func _TestIndividualFiles(t *testing.T) { +func TestIndividualFiles(t *testing.T) { config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) @@ -180,6 +181,10 @@ func _TestIndividualFiles(t *testing.T) { } for resource := range requestedClusterReports { + if !slices.Contains(resources, resource) { + continue + } + cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), "--resource", resource) if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) @@ -206,17 +211,16 @@ func _TestIndividualFiles(t *testing.T) { /* Tests for supported values of the --resource option */ -func _TestResourceOption(t *testing.T) { +func TestResourceOption(t *testing.T) { config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } testcmd := []string{"sh", "-c", "tar -tf cluster-dump.tar.gz --wildcards '*/summary.txt' 2>/dev/null | wc -l"} tests := []struct { - name string - resource string - want string - kubeconfig string + name string + resource string + want string }{ { name: "none", @@ -302,6 +306,15 @@ func _TestResourceOption(t *testing.T) { } } +type CmdCompare struct { + cmd []string + out string +} + +func PreareFindFileInTarCmd(tarPath, filePath, substring string) []string { + return []string{"tar", "--to-command", fmt.Sprintf("grep -m 1 -o %s", substring), "-xzf", tarPath, "--wildcards", filePath} +} + /* PT-2299 - collect openssl x509 certificate information for each secret */ @@ -313,117 +326,52 @@ func TestSSLResourceOption(t *testing.T) { tests := []struct { name string resource string - cmds [][]string // slice of commands to execute - want []string // slice of expected results + cmdOut []CmdCompare }{ { name: "auto pxc", resource: "auto", - cmds: [][]string{ - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-internal"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-internal"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-internal"}, - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - }, - want: []string{ - "ca.crt", - "Certificate", - "tls.crt", - "ca.crt", - "Certificate", - "tls.crt", - "ca.crt", - "Certificate", - "tls.crt", + cmdOut: []CmdCompare{ + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl", "ca.crt"), "ca.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl", "tls.crt"), "tls.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl-internal", "ca.crt"), "ca.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl-internal", "tls.crt"), "tls.crt"}, }, }, { name: "auto ps", resource: "auto", - cmds: [][]string{ - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - }, - want: []string{ - "ca.crt", - "Certificate", - "tls.crt", - "ca.crt", - "Certificate", - "tls.crt", + cmdOut: []CmdCompare{ + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl", "ca.crt"), "ca.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl", "tls.crt"), "tls.crt"}, }, }, { name: "auto psmdb", resource: "auto", - cmds: [][]string{ - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl"}, - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-internal"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-internal"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-internal"}, - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ca-cert"}, - }, - want: []string{ - "ca.crt", - "Certificate", - "tls.crt", - "ca.crt", - "Certificate", - "tls.crt", - "ca.crt", - "Certificate", - "tls.crt", + cmdOut: []CmdCompare{ + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl", "ca.crt"), "ca.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl", "tls.crt"), "tls.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl-internal", "ca.crt"), "ca.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl-internal", "tls.crt"), "tls.crt"}, }, }, { name: "auto pg", resource: "auto", - cmds: [][]string{ - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-ca"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-ca"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-keypair"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-ssl-keypair"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pgo.tls"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pgo.tls"}, - }, - want: []string{ - "ca.crt", - "Certificate", - "tls.crt\ntls.crt", - "Certificate\nCertificate", - "tls.crt", - "Certificate", + cmdOut: []CmdCompare{ + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl-keypair", "tls.crt"), "tls.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-pgo.tls", "tls.crt"), "tls.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ssl-ca", "ca.crt"), "ca.crt"}, }, }, { name: "auto pgv2", resource: "auto", - cmds: [][]string{ - {"tar", "--to-command", "grep -m 1 -o ca.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-cluster-cert"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-cluster-cert"}, - {"tar", "--to-command", "grep -m 1 -o tls.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*-cluster-cert"}, - {"tar", "--to-command", "grep -m 1 -o root.crt", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pgo-root-cacert"}, - {"tar", "--to-command", "grep -m 1 -o Certificate", "-xzf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pgo-root-cacert"}, - }, - want: []string{ - "ca.crt", - "Certificate", - "tls.crt", - "root.crt", - "Certificate", + cmdOut: []CmdCompare{ + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-ca-cert", "root.crt"), "root.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-cert", "tls.crt"), "tls.crt"}, + {PreareFindFileInTarCmd("cluster-dump.tar.gz", "cluster-dump/*/secrets/*-cert", "ca.crt"), "ca.crt"}, }, }, } @@ -436,19 +384,19 @@ func TestSSLResourceOption(t *testing.T) { if err := cmd.Run(); err != nil { t.Errorf("error executing pt-k8s-debug-collector: %s", err.Error()) } - // defer func() { - // cmd = exec.Command("rm", "-f", "cluster-dump.tar.gz") - // if err := cmd.Run(); err != nil { - // t.Errorf("error cleaning up test data: %s", err.Error()) - // } - // }() - for ind, testcmd := range test.cmds { - out, err := exec.Command(testcmd[0], testcmd[1:]...).Output() + defer func() { + cmd = exec.Command("rm", "-f", "cluster-dump.tar.gz") + if err := cmd.Run(); err != nil { + t.Errorf("error cleaning up test data: %s", err.Error()) + } + }() + for _, testcmd := range test.cmdOut { + out, err := exec.Command(testcmd.cmd[0], testcmd.cmd[1:]...).Output() if err != nil { t.Errorf("test %s, error running command %s:\n%s\n\nCommand output:\n%s", test.name, testcmd, err.Error(), out) } - if strings.TrimRight(bytes.NewBuffer(out).String(), "\n") != test.want[ind] { - t.Errorf("test %s, output is not as expected\nOutput: %s\nWanted: %s", test.name, out, test.want[ind]) + if strings.TrimRight(bytes.NewBuffer(out).String(), "\n") != testcmd.out { + t.Errorf("test %s, output is not as expected\nOutput: %s\nWanted: %s", test.name, out, testcmd.out) } } } @@ -553,7 +501,7 @@ func _TestPT_2453(t *testing.T) { /* Option --version */ -func __TestVersionOption(t *testing.T) { +func TestVersionOption(t *testing.T) { out, err := exec.Command("../../../bin/"+toolname, "--version").Output() if err != nil { t.Errorf("error executing %s --version: %s", toolname, err.Error()) @@ -568,14 +516,14 @@ func __TestVersionOption(t *testing.T) { /* If we handle error properly */ -func _TestPT_2169(t *testing.T) { +func TestPT_2169(t *testing.T) { config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) } busyport, _ := os.Getwd() // we are using wrong socket for ssh tunnel here to ensure we get error - testcmd := []string{"sh", "-c", "tar -xf cluster-dump.tar.gz --wildcards '*/summary.txt' --to-command 'grep stderr:' 2>/dev/null | wc -l"} + testcmd := []string{"sh", "-c", `tar -xf cluster-dump.tar.gz --wildcards "*/summary.txt" --to-command 'grep -m1 "err: strconv.ParseInt"' 2>/dev/null | wc -l`} tests := []struct { name string resource string diff --git a/src/go/tests/utils/cluster.go b/src/go/tests/utils/cluster.go index cc4e20e50..88b6f392a 100644 --- a/src/go/tests/utils/cluster.go +++ b/src/go/tests/utils/cluster.go @@ -71,7 +71,7 @@ func WaitForAllPodsReady( } for i, pod := range pods.Items { - if pod.Status.Phase != corev1.PodSucceeded { + if pod.Status.Phase == corev1.PodSucceeded { continue } From e0a73bb590348f82001993c0f130b367e240b672 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Wed, 14 Jan 2026 15:46:04 +0200 Subject: [PATCH 09/21] refactor --- .../pt-k8s-debug-collector/dumper/dumper.go | 1 + .../dumper/individual_files.go | 125 ++++++++++++------ .../dumper/kube_utils.go | 100 ++++++++++++++ .../dumper/resources.go | 9 ++ 4 files changed, 198 insertions(+), 37 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index 12aa2d5d3..f6c715822 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -54,6 +54,7 @@ type individualFile struct { resourceName string containerName string filepaths []string + dirpaths []string } // resourceMap struct is used to dump the resources from namespace scope or cluster scope diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index 66a141978..9ab94b928 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -2,68 +2,119 @@ package dumper import ( "archive/tar" - "bytes" "context" - "errors" "fmt" "io" "log" - - corev1 "k8s.io/api/core/v1" + "path" ) func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType string) { for _, indf := range d.individualFiles { - if indf.resourceName == crType { - for _, indPath := range indf.filepaths { - file, err := d.getFileFromPod(ctx, job.Pod, indPath, indf.containerName) - if err != nil { - log.Printf("error while getting individual files for %q pod and %q namespace to dump: %s, SKIPPING", job.Pod.Name, job.Pod.Namespace, err) - continue - } - - if len(file) != 0 { - log.Printf("Writing individual file with path %s to dump", indPath) - path := d.PodIndividualFilesPath(job.Pod.Namespace, job.Pod.Name, indPath) - err = d.archive.WriteVirtualFile(path, file) - if err != nil { - log.Printf("error while writing individual files for %q pod and %q namespace to dump: %s", job.Pod.Name, job.Pod.Namespace, err) - } - } + if indf.resourceName != crType { + continue + } + + var err error + for _, indPath := range indf.filepaths { + indPath, err = d.ParseEnvsFromSpec(ctx, job.Pod.Namespace, job.Pod.Name, indf.containerName, indPath) + if err != nil { + log.Printf("Skipping file %q. Failed to parse ENV's", indPath) + continue + } + if err := d.processSingleFile(ctx, job, indf.containerName, indPath); err != nil { + log.Printf("Skipping file %q: %v", indPath, err) + } + } + + for _, dirPath := range indf.dirpaths { + dirPath, err = d.ParseEnvsFromSpec(ctx, job.Pod.Namespace, job.Pod.Name, indf.containerName, dirPath) + if err != nil { + log.Printf("Skipping directory %q. Failed to parse ENV's", dirPath) + continue + } + + if err := d.processDir(ctx, job, indf.containerName, dirPath); err != nil { + log.Printf("Skipping directory %q: %v", dirPath, err) } } } } -func (d *Dumper) getFileFromPod(ctx context.Context, pod corev1.Pod, filepath, containerName string) ([]byte, error) { - if len(filepath) == 0 || len(containerName) == 0 { - return nil, errors.New("container name or filepath is not specified") - } +func (d *Dumper) processSingleFile( + ctx context.Context, + job exportJob, + container, filePath string, +) error { - cmd := []string{"tar", "cf", "-", filepath} - stdout, stderr, err := d.executeInPod(ctx, cmd, pod, containerName, nil) + tr, rc, stderr, err := d.tarFromPod(ctx, job.Pod, container, filePath) if err != nil { - return nil, fmt.Errorf("failed to execute command in Pod: stderr: %s: %w", &stderr, err) + return fmt.Errorf("exec tar: %w (stderr: %s)", err, stderr.String()) } + defer rc.Close() - tarReader := tar.NewReader(&stdout) - var fileContentBuffer bytes.Buffer for { - header, err := tarReader.Next() + hdr, err := tr.Next() if err == io.EOF { break } if err != nil { - return nil, fmt.Errorf("error reading tar header: %w", err) + return err } - if header.Typeflag == tar.TypeReg && header.Name == filepath { - _, copyErr := io.Copy(&fileContentBuffer, tarReader) - if copyErr != nil { - return nil, fmt.Errorf("error copying file content: %w", copyErr) - } + if hdr.Typeflag != tar.TypeReg { + continue + } + + if path.Base(hdr.Name) != path.Base(filePath) { + continue } + + dst := d.PodIndividualFilesPath( + job.Pod.Namespace, + job.Pod.Name, + path.Base(filePath), + ) + + return d.archive.WriteFile(dst, tr, hdr.Size) + } + + return fmt.Errorf("file %q not found", filePath) +} + +func (d *Dumper) processDir( + ctx context.Context, + job exportJob, + container, dir string, +) error { + + tr, rc, _, err := d.tarFromPod(ctx, job.Pod, container, "-C", dir, ".") + if err != nil { + return err } + defer rc.Close() + + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + dst := d.PodIndividualFilesPath( + job.Pod.Namespace, + job.Pod.Name, + path.Base(hdr.Name), + ) - return fileContentBuffer.Bytes(), nil + if err := d.archive.WriteFile(dst, tr, hdr.Size); err != nil { + return err + } + } } diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index b1bfa77e5..b0e4c55e0 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -1,6 +1,7 @@ package dumper import ( + "archive/tar" "bytes" "context" "errors" @@ -11,8 +12,10 @@ import ( "net/url" "path" "strconv" + "strings" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/tools/remotecommand" @@ -151,3 +154,100 @@ func (d *Dumper) executeInPod(ctx context.Context, command []string, pod corev1. return outb, errb, nil } + +func (d *Dumper) tarFromPod( + ctx context.Context, + pod corev1.Pod, + container string, + args ...string, +) (*tar.Reader, io.ReadCloser, *bytes.Buffer, error) { + + cmd := append([]string{"tar", "cf", "-"}, args...) + + stdout, stderr, err := d.executeInPodStream(ctx, cmd, pod, container, nil) + if err != nil { + return nil, nil, nil, err + } + + return tar.NewReader(stdout), stdout, &stderr, nil +} + +// DrainCloser wraps an io.ReadCloser to ensure proper closure of pod exec streams. +// Kubernetes SPDY transport may try to write to a closed pipe if stdout is closed +// before fully read, causing "io: read/write on closed pipe" logs. +// Close() drains the remaining data to io.Discard to avoid these errors. +type DrainCloser struct{ io.ReadCloser } + +func (d DrainCloser) Close() error { + if d.ReadCloser == nil { + return nil + } + io.Copy(io.Discard, d.ReadCloser) + err := d.ReadCloser.Close() + d.ReadCloser = nil + return err +} + +func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod corev1.Pod, container string, stdin io.Reader) (io.ReadCloser, bytes.Buffer, error) { + stdinFlag := stdin != nil + var stderr bytes.Buffer + + req := d.clientSet.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdin: stdinFlag, + Stdout: true, + Stderr: true, + TTY: false, + Container: container, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(d.restConfig, "POST", req.URL()) + if err != nil { + return nil, stderr, fmt.Errorf("error creating SPDY executor: %w", err) + } + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + + if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: pw, + Stderr: &stderr, + Tty: false, + }); err != nil && !errors.Is(err, context.Canceled) { + log.Printf("error while streaming files from pod: %s", err.Error()) + } + }() + + return DrainCloser{pr}, stderr, nil +} + +func (d *Dumper) ParseEnvsFromSpec(ctx context.Context, namespace, podName, container, input string) (string, error) { + if !strings.Contains(input, "$") { + return input, nil + } + + pod, err := d.clientSet.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return "", err + } + + for _, c := range pod.Spec.Containers { + if c.Name == container { + resolved := input + for _, e := range c.Env { + resolved = strings.ReplaceAll(resolved, "$"+e.Name, e.Value) + } + return resolved, nil + } + } + + return "", fmt.Errorf("container %s not found in pod %s", container, podName) +} diff --git a/src/go/pt-k8s-debug-collector/dumper/resources.go b/src/go/pt-k8s-debug-collector/dumper/resources.go index 766a618ef..0e8c05daa 100644 --- a/src/go/pt-k8s-debug-collector/dumper/resources.go +++ b/src/go/pt-k8s-debug-collector/dumper/resources.go @@ -11,6 +11,15 @@ import ( var resourcesRe = regexp.MustCompile(`(\w+\.(\w+).percona\.com)`) func (d *Dumper) addPg1() error { + dirpaths := []string{ + "$PGBACKREST_DB_PATH/pg_log", + } + + d.individualFiles = append(d.individualFiles, individualFile{ + resourceName: "pg", + containerName: "database", + dirpaths: dirpaths, + }) return nil } From df8e2ee5755ee936955718e28df1d28e310934f4 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Wed, 14 Jan 2026 15:51:03 +0200 Subject: [PATCH 10/21] requested fixes --- go.mod | 21 +------ go.sum | 59 +------------------ .../pt-k8s-debug-collector/dumper/dumper.go | 2 - .../pt-k8s-debug-collector/dumper/secrets.go | 2 +- .../pt-k8s-debug-collector/dumper/summary.go | 2 +- 5 files changed, 4 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index 988ff414f..beae5bf9b 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/percona/percona-toolkit -go 1.24.0 - -toolchain go1.24.1 +go 1.25.5 require ( github.com/AlekSi/pointer v1.2.0 @@ -44,16 +42,8 @@ require ( require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect - github.com/cilium/ebpf v0.20.0 // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect - github.com/cosiner/argv v0.1.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect - github.com/derekparker/trie/v3 v3.2.1 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-delve/delve v1.26.0 // indirect - github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -62,25 +52,19 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/go-dap v0.12.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/rivo/uniseg v0.4.7 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect @@ -90,14 +74,11 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.starlark.net v0.0.0-20260102030733-3fee463870c9 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.23.0 // indirect golang.org/x/net v0.46.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.39.0 // indirect - golang.org/x/telemetry v0.0.0-20251222180846-3f2a21fb04ff // indirect golang.org/x/term v0.36.0 // indirect golang.org/x/text v0.30.0 // indirect golang.org/x/time v0.9.0 // indirect diff --git a/go.sum b/go.sum index 4cc56d0a8..06fcf4506 100644 --- a/go.sum +++ b/go.sum @@ -18,33 +18,16 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/cilium/ebpf v0.20.0 h1:atwWj9d3NffHyPZzVlx3hmw1on5CLe9eljR8VuHTwhM= -github.com/cilium/ebpf v0.20.0/go.mod h1:pzLjFymM+uZPLk/IXZUL63xdx5VXEo+enTzxkZXdycw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= -github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= -github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/derekparker/trie/v3 v3.2.1 h1:fkW2422T+lmRCKD7zuQ97MPgKETXkmJGa0Uze3+5nfU= -github.com/derekparker/trie/v3 v3.2.1/go.mod h1:P94lW0LPgiaMgKAEQD59IDZD2jMK9paKok8Nli/nQbE= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-delve/delve v1.26.0 h1:YZT1kXD76mxba4/wr+tyUa/tSmy7qzoDsmxutT42PIs= -github.com/go-delve/delve v1.26.0/go.mod h1:8BgFFOXTi1y1M+d/4ax1LdFw0mlqezQiTZQpbpwgBxo= -github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62 h1:IGtvsNyIuRjl04XAOFGACozgUD7A82UffYxZt4DWbvA= -github.com/go-delve/liner v1.2.3-0.20231231155935-4726ab1d7f62/go.mod h1:biJCRbqp51wS+I92HMqn5H8/A0PAhxn2vyOT+JqhiGI= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -74,8 +57,6 @@ github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnL github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-dap v0.12.0 h1:rVcjv3SyMIrpaOoTAdFDyHs99CwVOItIJGKLQFQhNeM= -github.com/google/go-dap v0.12.0/go.mod h1:tNjCASCm5cqePi/RVXXWEVqtnNLV1KTWtYOqu6rZNzc= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= @@ -89,8 +70,6 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef h1:A9HsByNhogrvm9cWb28sjiS3i7tcKCkflWFEkHfuAgM= github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -110,18 +89,13 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= @@ -151,24 +125,15 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= -github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= -github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -208,26 +173,18 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.starlark.net v0.0.0-20260102030733-3fee463870c9 h1:nV1OyvU+0CYrp5eKfQ3rD03TpFYYhH08z31NK1HmtTk= -go.starlark.net v0.0.0-20260102030733-3fee463870c9/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= -golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -241,8 +198,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= @@ -252,8 +207,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -264,7 +217,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -272,24 +224,16 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20251222180846-3f2a21fb04ff h1:1QaeZGjxSnF1KOGnUYQmI1YpaBe0FvBE1K2rRDuxawc= -golang.org/x/telemetry v0.0.0-20251222180846-3f2a21fb04ff/go.mod h1:ArQvPJS723nJQietgilmZA+shuB3CZxH1n2iXq9VSfs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= @@ -300,9 +244,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index 12aa2d5d3..7c2e7444c 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -117,8 +117,6 @@ func New(location, namespace, kubeconfig, forwardport, resource string, skipPodS return nil, err } - log.Printf("DBG: FOUND CR's: %v", d.crTypes) - d.sslSecrets = make(map[string]bool, 0) for _, cr := range d.crTypes { switch resourceType(cr) { diff --git a/src/go/pt-k8s-debug-collector/dumper/secrets.go b/src/go/pt-k8s-debug-collector/dumper/secrets.go index 14bf77c92..9305985ff 100644 --- a/src/go/pt-k8s-debug-collector/dumper/secrets.go +++ b/src/go/pt-k8s-debug-collector/dumper/secrets.go @@ -122,5 +122,5 @@ func (d *Dumper) getSecretValueFromPod(ctx context.Context, pod corev1.Pod, secr return string(secretValueBytes), nil } } - return "", fmt.Errorf("could not find any secret with name %s in Pod '%s/%s", secretName, pod.Namespace, pod.Name) + return "", fmt.Errorf("could not find any secret with name %s in Pod '%s/%s'", secretName, pod.Namespace, pod.Name) } diff --git a/src/go/pt-k8s-debug-collector/dumper/summary.go b/src/go/pt-k8s-debug-collector/dumper/summary.go index a95522e70..718c3dc8b 100644 --- a/src/go/pt-k8s-debug-collector/dumper/summary.go +++ b/src/go/pt-k8s-debug-collector/dumper/summary.go @@ -22,7 +22,7 @@ func (d *Dumper) getSummary(ctx context.Context, job exportJob, crType string, l log.Printf("Error: create summary errors archive for pod %s in namespace %s: %v", job.Pod.Name, job.Pod.Namespace, err) } } else { - log.Printf("Created summary for pod/namespace %q/%q, Writing to dump", job.Pod.Name, job.Pod.Namespace) + log.Printf("Created summary for / %s/%s, Writing to dump", job.Pod.Name, job.Pod.Namespace) err = d.archive.WriteVirtualFile(location, output) if err != nil { log.Printf("error while writing summary for %q pod and %q namespace to dump: %s", job.Pod.Name, job.Pod.Namespace, err) From 960ab165f010046a369cd1ac06c515d2970f9551 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 15 Jan 2026 14:14:32 +0200 Subject: [PATCH 11/21] fix tests --- src/go/pt-k8s-debug-collector/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 98f26e4a3..5c0765a35 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -229,7 +229,7 @@ func TestIndividualFiles(t *testing.T) { // if the tool collects required pg log files name: "pg_logs_list", resource: "pg", - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/pgdata/*"}, + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/*"}, preprocessor: uniqueBasenames, match: RegexMatch{ Pattern: regexp.MustCompile(`^postgresql-[A-Za-z]{3}\.log$`), From 440e65d31033a75db0b6fd40b2e359787468e9a5 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 15 Jan 2026 15:04:51 +0200 Subject: [PATCH 12/21] fix tests --- src/go/pt-k8s-debug-collector/main_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 5c0765a35..acd8a92a1 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -243,9 +243,6 @@ func TestIndividualFiles(t *testing.T) { } for resource := range requestedClusterReports { - if resource != "pg" { - continue - } cmd := exec.Command("../../../bin/pt-k8s-debug-collector", "--kubeconfig", config, "--forwardport", os.Getenv("FORWARDPORT"), @@ -262,6 +259,10 @@ func TestIndividualFiles(t *testing.T) { }() for _, test := range tests { + if !slices.Contains(resources, test.resource) { + continue + } + out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput() if err != nil { t.Errorf("test %s, error running command %s:\n%s\nOutput:\n%s", test.name, test.cmd[0], err.Error(), out) From 022ba5fcfaf3518528542725bb7555d85a92ddec Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 15 Jan 2026 15:11:42 +0200 Subject: [PATCH 13/21] refactor --- .../pt-k8s-debug-collector/dumper/dumper.go | 2 +- .../dumper/individual_files.go | 30 ++++++++++--------- .../dumper/resources.go | 4 +-- src/go/pt-k8s-debug-collector/main_test.go | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index fa8457a5d..b4e9a1870 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -54,7 +54,7 @@ type individualFile struct { resourceName string containerName string filepaths []string - dirpaths []string + dirpaths map[string][]string } // resourceMap struct is used to dump the resources from namespace scope or cluster scope diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index 9ab94b928..4b21360da 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -22,20 +22,22 @@ func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType s log.Printf("Skipping file %q. Failed to parse ENV's", indPath) continue } - if err := d.processSingleFile(ctx, job, indf.containerName, indPath); err != nil { + if err := d.processSingleFile(ctx, job, indf.containerName, "", indPath); err != nil { log.Printf("Skipping file %q: %v", indPath, err) } } - for _, dirPath := range indf.dirpaths { - dirPath, err = d.ParseEnvsFromSpec(ctx, job.Pod.Namespace, job.Pod.Name, indf.containerName, dirPath) - if err != nil { - log.Printf("Skipping directory %q. Failed to parse ENV's", dirPath) - continue - } - - if err := d.processDir(ctx, job, indf.containerName, dirPath); err != nil { - log.Printf("Skipping directory %q: %v", dirPath, err) + for tarFolder, dirPaths := range indf.dirpaths { + for _, dirPath := range dirPaths { + dirPath, err = d.ParseEnvsFromSpec(ctx, job.Pod.Namespace, job.Pod.Name, indf.containerName, dirPath) + if err != nil { + log.Printf("Skipping directory %q. Failed to parse ENV's", dirPath) + continue + } + + if err := d.processDir(ctx, job, indf.containerName, tarFolder, dirPath); err != nil { + log.Printf("Skipping directory %q: %v", dirPath, err) + } } } } @@ -44,7 +46,7 @@ func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType s func (d *Dumper) processSingleFile( ctx context.Context, job exportJob, - container, filePath string, + container, tarFolder, filePath string, ) error { tr, rc, stderr, err := d.tarFromPod(ctx, job.Pod, container, filePath) @@ -73,7 +75,7 @@ func (d *Dumper) processSingleFile( dst := d.PodIndividualFilesPath( job.Pod.Namespace, job.Pod.Name, - path.Base(filePath), + path.Join(tarFolder, path.Base(filePath)), ) return d.archive.WriteFile(dst, tr, hdr.Size) @@ -85,7 +87,7 @@ func (d *Dumper) processSingleFile( func (d *Dumper) processDir( ctx context.Context, job exportJob, - container, dir string, + container, tarFolder, dir string, ) error { tr, rc, _, err := d.tarFromPod(ctx, job.Pod, container, "-C", dir, ".") @@ -110,7 +112,7 @@ func (d *Dumper) processDir( dst := d.PodIndividualFilesPath( job.Pod.Namespace, job.Pod.Name, - path.Base(hdr.Name), + path.Join(tarFolder, path.Base(hdr.Name)), ) if err := d.archive.WriteFile(dst, tr, hdr.Size); err != nil { diff --git a/src/go/pt-k8s-debug-collector/dumper/resources.go b/src/go/pt-k8s-debug-collector/dumper/resources.go index 0e8c05daa..a186e1e77 100644 --- a/src/go/pt-k8s-debug-collector/dumper/resources.go +++ b/src/go/pt-k8s-debug-collector/dumper/resources.go @@ -11,8 +11,8 @@ import ( var resourcesRe = regexp.MustCompile(`(\w+\.(\w+).percona\.com)`) func (d *Dumper) addPg1() error { - dirpaths := []string{ - "$PGBACKREST_DB_PATH/pg_log", + dirpaths := map[string][]string{ + "pg_log": {"$PGBACKREST_DB_PATH/pg_log"}, } d.individualFiles = append(d.individualFiles, individualFile{ diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index acd8a92a1..288c624a3 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -229,7 +229,7 @@ func TestIndividualFiles(t *testing.T) { // if the tool collects required pg log files name: "pg_logs_list", resource: "pg", - cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/*"}, + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/*/pg_log/*"}, preprocessor: uniqueBasenames, match: RegexMatch{ Pattern: regexp.MustCompile(`^postgresql-[A-Za-z]{3}\.log$`), From 0bf0037f21538c321e26ca522bcd7e8b51d21d21 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Thu, 15 Jan 2026 16:04:38 +0200 Subject: [PATCH 14/21] fix tests --- src/go/pt-k8s-debug-collector/main_test.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 288c624a3..7f76edcfd 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -33,18 +33,15 @@ This test requires: */ var ( namespaces = []string{ - //"pxc", "ps", "psmdb", "pg", "pgv2", - "pg", + "pxc", "ps", "psmdb", "pg", "pgv2", } resources = []string{ - //"pxc", "ps", "psmdb", "pg", "pgv2", "auto", "none", - "pg", "none", "auto", + "pxc", "ps", "psmdb", "pg", "pgv2", "auto", "none", } deployments = []string{ - //"k8s-pxc:1.18.0", "k8s-ps:1.0.0", "k8s-psmdb:1.21.1", "k8s-pg:1.6.0", "k8s-pg:2.8.0", - "k8s-pg:1.6.0", + "k8s-pxc:1.18.0", "k8s-ps:1.0.0", "k8s-psmdb:1.21.1", "k8s-pg:1.6.0", "k8s-pg:2.8.0", } ) @@ -137,7 +134,7 @@ func TestMain(m *testing.M) { log.Println("START") args := []string{"deploy"} args = append(args, deployments...) - //utils.DeployAnyDbVer(ctx, args) + utils.DeployAnyDbVer(ctx, args) config, err := utils.GetKubeConfigPath() if err != nil { @@ -167,10 +164,10 @@ func TestMain(m *testing.M) { if exitCode == 0 { log.Println("Tests finished succesfully, destroying deployments") // Comment this if you don't want to destroy deployments after tests - // err := utils.CleanUpAnyDbVer(ctx) - // if err != nil { - // log.Fatalf("there was an error when destroying deloyments: %v", err) - // } + err := utils.CleanUpAnyDbVer(ctx) + if err != nil { + log.Fatalf("there was an error when destroying deloyments: %v", err) + } } os.Exit(exitCode) } @@ -471,7 +468,7 @@ func TestSSLResourceOption(t *testing.T) { /* Tests for option --skip-pod-summary */ -func _TestPT_2453(t *testing.T) { +func TestPT_2453(t *testing.T) { config, err := utils.GetKubeConfigPath() if err != nil { t.Fatalf("error getting config for kube: %v", err) From d07a5bc8a8edf56641f414e414f54692d81f7e79 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Tue, 31 Mar 2026 13:28:07 +0300 Subject: [PATCH 15/21] Fix confilicts --- .../pt-k8s-debug-collector/dumper/dumper.go | 1 + .../dumper/individual_files.go | 152 ++++++++++++++---- .../dumper/kube_utils.go | 102 ++++++++++++ .../dumper/resources.go | 18 +++ src/go/pt-k8s-debug-collector/main_test.go | 59 ++++++- 5 files changed, 295 insertions(+), 37 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index 3303e9fbe..baaca1b15 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -70,6 +70,7 @@ type individualFile struct { resourceName string containerName string filepaths []string + dirpaths map[string][]string // map[tarFolder][]dirPaths } // resourceMap struct is used to dump the resources from namespace scope or cluster scope diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index 046b8b020..6f670310b 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -2,69 +2,153 @@ package dumper import ( "archive/tar" - "bytes" "context" - "errors" "fmt" "io" + "path" + "strings" log "github.com/sirupsen/logrus" - corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// getContainerEnvMap parses environment variables from pod container spec once +func (d *Dumper) getContainerEnvMap(ctx context.Context, namespace, podName, containerName string) (map[string]string, error) { + pod, err := d.clientSet.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + envMap := make(map[string]string) + for _, c := range pod.Spec.Containers { + if c.Name == containerName { + for _, e := range c.Env { + envMap[e.Name] = e.Value + } + return envMap, nil + } + } + + return nil, fmt.Errorf("container %s not found in pod %s/%s", containerName, namespace, podName) +} + +// replaceEnvVars replaces environment variables in input using provided env map +func replaceEnvVars(input string, envMap map[string]string) string { + result := input + for envName, envValue := range envMap { + result = strings.ReplaceAll(result, "$"+envName, envValue) + } + return result +} + func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType string) { for _, indf := range d.individualFiles { - if indf.resourceName == crType { - for _, indPath := range indf.filepaths { - file, err := d.getFileFromPod(ctx, job.Pod, indPath, indf.containerName) - if err != nil { - log.Infof("skipping file dump for %s/%s due to error: %s", job.Pod.Namespace, job.Pod.Name, err) - continue - } + if indf.resourceName != crType { + continue + } + + // Parse environment variables once for this container + envMap, err := d.getContainerEnvMap(ctx, job.Pod.Namespace, job.Pod.Name, indf.containerName) + if err != nil { + log.Warnf("Failed to get env for container %q: %v", indf.containerName, err) + continue + } - if len(file) != 0 { - log.Infof("pod: %q writing individual file with path %s to dump", job.Pod.Name, indPath) - path := d.PodIndividualFilesPath(job.Pod.Namespace, job.Pod.Name, indPath) - err = d.archive.WriteVirtualFile(path, file) - if err != nil { - log.Errorf("error while dumping individual files for %s/%s: %s", job.Pod.Namespace, job.Pod.Name, err) - } + // Process individual files + for _, indPath := range indf.filepaths { + resolvedPath := replaceEnvVars(indPath, envMap) + if err := d.processSingleFile(ctx, job, indf.containerName, "", resolvedPath); err != nil { + log.Warnf("Failed to process file %q: %v", resolvedPath, err) + } + } + + // Process directories + for tarFolder, dirPaths := range indf.dirpaths { + for _, dirPath := range dirPaths { + resolvedPath := replaceEnvVars(dirPath, envMap) + if err := d.processDir(ctx, job, indf.containerName, tarFolder, resolvedPath); err != nil { + log.Warnf("Skipping directory %q: %v", resolvedPath, err) } } } } } -func (d *Dumper) getFileFromPod(ctx context.Context, pod corev1.Pod, filepath, containerName string) ([]byte, error) { - if len(filepath) == 0 || len(containerName) == 0 { - return nil, errors.New("container name or filepath is not specified") - } +func (d *Dumper) processSingleFile( + ctx context.Context, + job exportJob, + container, tarFolder, filePath string, +) error { - cmd := []string{"tar", "cf", "-", filepath} - stdout, stderr, err := d.executeInPod(ctx, cmd, pod, containerName, nil) + tr, rc, stderr, err := d.tarFromPod(ctx, job.Pod, container, filePath) if err != nil { - return nil, fmt.Errorf("failed to execute command in Pod: stderr: %s: %w", &stderr, err) + return fmt.Errorf("exec tar: %w (stderr: %s)", err, stderr.String()) } + defer rc.Close() - tarReader := tar.NewReader(&stdout) - var fileContentBuffer bytes.Buffer for { - header, err := tarReader.Next() + hdr, err := tr.Next() if err == io.EOF { break } if err != nil { - return nil, fmt.Errorf("error reading tar header: %w", err) + return err } - if header.Typeflag == tar.TypeReg && header.Name == filepath { - _, copyErr := io.Copy(&fileContentBuffer, tarReader) - if copyErr != nil { - return nil, fmt.Errorf("error copying file content: %w", copyErr) - } + if hdr.Typeflag != tar.TypeReg { + continue } + + if path.Base(hdr.Name) != path.Base(filePath) { + continue + } + + dst := d.PodIndividualFilesPath( + job.Pod.Namespace, + job.Pod.Name, + path.Join(tarFolder, path.Base(filePath)), + ) + + return d.archive.WriteFile(dst, tr, hdr.Size) } - return fileContentBuffer.Bytes(), nil + return fmt.Errorf("file %q not found", filePath) +} + +func (d *Dumper) processDir( + ctx context.Context, + job exportJob, + container, tarFolder, dir string, +) error { + + tr, rc, _, err := d.tarFromPod(ctx, job.Pod, container, "-C", dir, ".") + if err != nil { + return err + } + defer rc.Close() + + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + dst := d.PodIndividualFilesPath( + job.Pod.Namespace, + job.Pod.Name, + path.Join(tarFolder, path.Base(hdr.Name)), + ) + + if err := d.archive.WriteFile(dst, tr, hdr.Size); err != nil { + return err + } + } } diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index 5cead2994..6181db1cc 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -1,6 +1,7 @@ package dumper import ( + "archive/tar" "bytes" "context" "errors" @@ -10,10 +11,12 @@ import ( "net/url" "path" "strconv" + "strings" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/tools/remotecommand" @@ -153,3 +156,102 @@ func (d *Dumper) executeInPod(ctx context.Context, command []string, pod corev1. return outb, errb, nil } + +// tarFromPod executes tar command in pod and returns tar reader, read closer, and stderr +func (d *Dumper) tarFromPod( + ctx context.Context, + pod corev1.Pod, + container string, + args ...string, +) (*tar.Reader, io.ReadCloser, *bytes.Buffer, error) { + cmd := append([]string{"tar", "cf", "-"}, args...) + + stdout, stderr, err := d.executeInPodStream(ctx, cmd, pod, container, nil) + if err != nil { + return nil, nil, nil, err + } + + return tar.NewReader(stdout), stdout, &stderr, nil +} + +// DrainCloser wraps an io.ReadCloser to ensure proper closure of pod exec streams. +// Kubernetes SPDY transport may try to write to a closed pipe if stdout is closed +// before fully read, causing "io: read/write on closed pipe" logs. +// Close() drains the remaining data to io.Discard to avoid these errors. +type DrainCloser struct{ io.ReadCloser } + +func (d DrainCloser) Close() error { + if d.ReadCloser == nil { + return nil + } + io.Copy(io.Discard, d.ReadCloser) + err := d.ReadCloser.Close() + d.ReadCloser = nil + return err +} + +// executeInPodStream executes command in pod and streams the output +func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod corev1.Pod, container string, stdin io.Reader) (io.ReadCloser, bytes.Buffer, error) { + stdinFlag := stdin != nil + var stderr bytes.Buffer + + req := d.clientSet.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Command: command, + Stdin: stdinFlag, + Stdout: true, + Stderr: true, + TTY: false, + Container: container, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(d.restConfig, "POST", req.URL()) + if err != nil { + return nil, stderr, fmt.Errorf("error creating SPDY executor: %w", err) + } + + pr, pw := io.Pipe() + + go func() { + defer pw.Close() + + if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: pw, + Stderr: &stderr, + Tty: false, + }); err != nil && !errors.Is(err, context.Canceled) { + log.Errorf("error while streaming files from pod: %s", err.Error()) + } + }() + + return DrainCloser{pr}, stderr, nil +} + +// ParseEnvsFromSpec parses environment variables in input string +func (d *Dumper) ParseEnvsFromSpec(ctx context.Context, namespace, podName, container, input string) (string, error) { + if !strings.Contains(input, "$") { + return input, nil + } + + pod, err := d.clientSet.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return "", err + } + + for _, c := range pod.Spec.Containers { + if c.Name == container { + resolved := input + for _, e := range c.Env { + resolved = strings.ReplaceAll(resolved, "$"+e.Name, e.Value) + } + return resolved, nil + } + } + + return "", fmt.Errorf("container %s not found in pod %s", container, podName) +} diff --git a/src/go/pt-k8s-debug-collector/dumper/resources.go b/src/go/pt-k8s-debug-collector/dumper/resources.go index 04d939d7e..95e482943 100644 --- a/src/go/pt-k8s-debug-collector/dumper/resources.go +++ b/src/go/pt-k8s-debug-collector/dumper/resources.go @@ -12,10 +12,28 @@ import ( var resourcesRe = regexp.MustCompile(`(\w+\.(\w+).percona\.com)`) func (d *Dumper) addPg1() error { + dirpaths := map[string][]string{ + "pg_log": {"$PGBACKREST_DB_PATH/pg_log"}, + } + + d.individualFiles = append(d.individualFiles, individualFile{ + resourceName: "pgo", + containerName: "database", + dirpaths: dirpaths, + }) return nil } func (d *Dumper) addPg2() error { + dirpaths := map[string][]string{ + "pg_log": {"$PGDATA/log"}, + } + + d.individualFiles = append(d.individualFiles, individualFile{ + resourceName: "pgv2", + containerName: "database", + dirpaths: dirpaths, + }) return nil } diff --git a/src/go/pt-k8s-debug-collector/main_test.go b/src/go/pt-k8s-debug-collector/main_test.go index 834018645..e6cf4b934 100644 --- a/src/go/pt-k8s-debug-collector/main_test.go +++ b/src/go/pt-k8s-debug-collector/main_test.go @@ -328,10 +328,63 @@ func (s *CollectorSuite) TestIndividualFiles() { return in[:nl] }, }, + { + namespace: "pgo", + // If the tool collects PostgreSQL log files + name: "pgo_pg_logs_exist", + // tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/*/pg_log/*.log' + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"}, + want: []string{".log"}, + preprocessor: func(in string) string { + files := strings.Split(in, "\n") + var result []string + for _, f := range files { + if strings.Contains(f, "pg_log") && strings.HasSuffix(f, ".log") { + result = append(result, ".log") + break // Just check if at least one .log file exists + } + } + return strings.Join(result, "") + }, + }, + { + namespace: "pgv2", + // If the tool collects PostgreSQL log files for pgv2 + name: "pgv2_pg_logs_exist", + // tar -tf cluster-dump.tar.gz --wildcards 'cluster-dump/*/pg_log/*.log' + cmd: []string{"tar", "-tf", "cluster-dump.tar.gz", "--wildcards", "cluster-dump/*/pg_log/*.log"}, + want: []string{".log"}, + preprocessor: func(in string) string { + files := strings.Split(in, "\n") + var result []string + for _, f := range files { + if strings.Contains(f, "pg_log") && strings.HasSuffix(f, ".log") { + result = append(result, ".log") + break // Just check if at least one .log file exists + } + } + return strings.Join(result, "") + }, + }, } - if s.Namespace != "pxc" { - s.T().Skip("This test is specifically for pxc namespace") + // Filter tests for current namespace + nsTests := []struct { + namespace string + name string + cmd []string + want []string + preprocessor func(string) string + }{} + + for _, test := range tests { + if test.namespace == s.Namespace { + nsTests = append(nsTests, test) + } + } + + if len(nsTests) == 0 { + s.T().Skip("No tests configured for namespace " + s.Namespace) } for _, resource := range s.Resources { @@ -340,7 +393,7 @@ func (s *CollectorSuite) TestIndividualFiles() { err := cmd.Run() s.NoError(err) - for _, test := range tests { + for _, test := range nsTests { out, err := exec.Command(test.cmd[0], test.cmd[1:]...).CombinedOutput() if err != nil && resource == "none" { continue From 465ad9c58cf69884fa3400d1029634c05da157c0 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Wed, 1 Apr 2026 00:36:55 +0300 Subject: [PATCH 16/21] Fix pgo log export --- src/go/pt-k8s-debug-collector/dumper/dumper.go | 8 +++++--- src/go/pt-k8s-debug-collector/dumper/individual_files.go | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/dumper.go b/src/go/pt-k8s-debug-collector/dumper/dumper.go index baaca1b15..00cee465e 100644 --- a/src/go/pt-k8s-debug-collector/dumper/dumper.go +++ b/src/go/pt-k8s-debug-collector/dumper/dumper.go @@ -505,15 +505,17 @@ func matchesCR(cr string, podLabels map[string]string) bool { func (d *Dumper) exportPodSummaryAndFiles(ctx context.Context, job exportJob) { for _, cr := range d.crTypes { - if !matchesCR(cr, job.Pod.Labels) { + normalizedCR := resourceType(cr) + + if !matchesCR(normalizedCR, job.Pod.Labels) { continue } if !d.skipPodSummary { - d.getSummary(ctx, job, cr, d.PodSummaryPath(job.Pod.Namespace, job.Pod.Name)) + d.getSummary(ctx, job, normalizedCR, d.PodSummaryPath(job.Pod.Namespace, job.Pod.Name)) } - d.getIndividualFiles(ctx, job, cr) + d.getIndividualFiles(ctx, job, normalizedCR) } } diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index 6f670310b..fa3a18906 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -43,8 +43,10 @@ func replaceEnvVars(input string, envMap map[string]string) string { } func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType string) { + normalizedCRType := resourceType(crType) + for _, indf := range d.individualFiles { - if indf.resourceName != crType { + if resourceType(indf.resourceName) != normalizedCRType { continue } From cc80e735e8db8905e8292c739a7714d2c94ec388 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 3 Apr 2026 16:57:16 +0300 Subject: [PATCH 17/21] Fix error buffer --- .../dumper/individual_files.go | 6 ++--- .../dumper/kube_utils.go | 26 ++++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index fa3a18906..5286ed7aa 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -83,9 +83,9 @@ func (d *Dumper) processSingleFile( container, tarFolder, filePath string, ) error { - tr, rc, stderr, err := d.tarFromPod(ctx, job.Pod, container, filePath) + tr, rc, err := d.tarFromPod(ctx, job.Pod, container, filePath) if err != nil { - return fmt.Errorf("exec tar: %w (stderr: %s)", err, stderr.String()) + return fmt.Errorf("exec tar: %w", err) } defer rc.Close() @@ -124,7 +124,7 @@ func (d *Dumper) processDir( container, tarFolder, dir string, ) error { - tr, rc, _, err := d.tarFromPod(ctx, job.Pod, container, "-C", dir, ".") + tr, rc, err := d.tarFromPod(ctx, job.Pod, container, "-C", dir, ".") if err != nil { return err } diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index 6181db1cc..e37bb5735 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -157,21 +157,21 @@ func (d *Dumper) executeInPod(ctx context.Context, command []string, pod corev1. return outb, errb, nil } -// tarFromPod executes tar command in pod and returns tar reader, read closer, and stderr +// tarFromPod executes tar command in pod and returns tar reader and read closer. func (d *Dumper) tarFromPod( ctx context.Context, pod corev1.Pod, container string, args ...string, -) (*tar.Reader, io.ReadCloser, *bytes.Buffer, error) { +) (*tar.Reader, io.ReadCloser, error) { cmd := append([]string{"tar", "cf", "-"}, args...) - stdout, stderr, err := d.executeInPodStream(ctx, cmd, pod, container, nil) + stdout, err := d.executeInPodStream(ctx, cmd, pod, container, nil) if err != nil { - return nil, nil, nil, err + return nil, nil, err } - return tar.NewReader(stdout), stdout, &stderr, nil + return tar.NewReader(stdout), stdout, nil } // DrainCloser wraps an io.ReadCloser to ensure proper closure of pod exec streams. @@ -190,8 +190,10 @@ func (d DrainCloser) Close() error { return err } -// executeInPodStream executes command in pod and streams the output -func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod corev1.Pod, container string, stdin io.Reader) (io.ReadCloser, bytes.Buffer, error) { +// executeInPodStream executes command in pod and streams the output. +// Streaming errors are logged from the background goroutine because they can +// happen after this function has already returned the stdout reader. +func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod corev1.Pod, container string, stdin io.Reader) (io.ReadCloser, error) { stdinFlag := stdin != nil var stderr bytes.Buffer @@ -211,7 +213,7 @@ func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod c exec, err := remotecommand.NewSPDYExecutor(d.restConfig, "POST", req.URL()) if err != nil { - return nil, stderr, fmt.Errorf("error creating SPDY executor: %w", err) + return nil, fmt.Errorf("error creating SPDY executor: %w", err) } pr, pw := io.Pipe() @@ -225,11 +227,15 @@ func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod c Stderr: &stderr, Tty: false, }); err != nil && !errors.Is(err, context.Canceled) { - log.Errorf("error while streaming files from pod: %s", err.Error()) + if stderr.Len() > 0 { + log.Errorf("error while streaming files from pod: %s (stderr: %s)", err, stderr.String()) + return + } + log.Errorf("error while streaming files from pod: %s", err) } }() - return DrainCloser{pr}, stderr, nil + return DrainCloser{pr}, nil } // ParseEnvsFromSpec parses environment variables in input string From 31391744a6f1e7d8bcf5fafaacb58207acb6ed72 Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 3 Apr 2026 17:25:57 +0300 Subject: [PATCH 18/21] Fix error handling in gorutine --- src/go/pt-k8s-debug-collector/dumper/kube_utils.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go index e37bb5735..195bda474 100644 --- a/src/go/pt-k8s-debug-collector/dumper/kube_utils.go +++ b/src/go/pt-k8s-debug-collector/dumper/kube_utils.go @@ -184,7 +184,7 @@ func (d DrainCloser) Close() error { if d.ReadCloser == nil { return nil } - io.Copy(io.Discard, d.ReadCloser) + _, _ = io.Copy(io.Discard, d.ReadCloser) err := d.ReadCloser.Close() d.ReadCloser = nil return err @@ -219,8 +219,6 @@ func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod c pr, pw := io.Pipe() go func() { - defer pw.Close() - if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ Stdin: stdin, Stdout: pw, @@ -229,10 +227,15 @@ func (d *Dumper) executeInPodStream(ctx context.Context, command []string, pod c }); err != nil && !errors.Is(err, context.Canceled) { if stderr.Len() > 0 { log.Errorf("error while streaming files from pod: %s (stderr: %s)", err, stderr.String()) + _ = pw.CloseWithError(fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String()))) return } log.Errorf("error while streaming files from pod: %s", err) + _ = pw.CloseWithError(err) + return } + + _ = pw.Close() }() return DrainCloser{pr}, nil From 08463c614a0411bc4449e760b4b92d823c4c618a Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 3 Apr 2026 17:32:32 +0300 Subject: [PATCH 19/21] Fixes --- .../dumper/individual_files.go | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index 5286ed7aa..3001a039d 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -9,17 +9,11 @@ import ( "strings" log "github.com/sirupsen/logrus" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" ) // getContainerEnvMap parses environment variables from pod container spec once -func (d *Dumper) getContainerEnvMap(ctx context.Context, namespace, podName, containerName string) (map[string]string, error) { - pod, err := d.clientSet.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - +func (d *Dumper) getContainerEnvMap(pod corev1.Pod, containerName string) (map[string]string, error) { envMap := make(map[string]string) for _, c := range pod.Spec.Containers { if c.Name == containerName { @@ -30,7 +24,7 @@ func (d *Dumper) getContainerEnvMap(ctx context.Context, namespace, podName, con } } - return nil, fmt.Errorf("container %s not found in pod %s/%s", containerName, namespace, podName) + return nil, fmt.Errorf("container %s not found in pod %s/%s", containerName, pod.Namespace, pod.Name) } // replaceEnvVars replaces environment variables in input using provided env map @@ -51,7 +45,7 @@ func (d *Dumper) getIndividualFiles(ctx context.Context, job exportJob, crType s } // Parse environment variables once for this container - envMap, err := d.getContainerEnvMap(ctx, job.Pod.Namespace, job.Pod.Name, indf.containerName) + envMap, err := d.getContainerEnvMap(job.Pod, indf.containerName) if err != nil { log.Warnf("Failed to get env for container %q: %v", indf.containerName, err) continue @@ -143,6 +137,20 @@ func (d *Dumper) processDir( continue } + // Preserve the relative path from the tar header while ensuring it + // cannot escape the intended destination directory. + relPath := path.Clean(hdr.Name) + // Normalize common tar prefixes like "./" + relPath = strings.TrimPrefix(relPath, "./") + // Prevent path traversal outside tarFolder by stripping leading "../" + for strings.HasPrefix(relPath, "../") { + relPath = strings.TrimPrefix(relPath, "../") + } + // Skip entries that do not resolve to a meaningful relative path + if relPath == "" || relPath == "." { + continue + } + dst := d.PodIndividualFilesPath( job.Pod.Namespace, job.Pod.Name, From a8d74c09631e5f6e4837234fe819115917d01d1a Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 3 Apr 2026 17:46:31 +0300 Subject: [PATCH 20/21] Fix version --- src/go/pt-k8s-debug-collector/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/go/pt-k8s-debug-collector/main.go b/src/go/pt-k8s-debug-collector/main.go index f31ccd5d6..3e86063ee 100644 --- a/src/go/pt-k8s-debug-collector/main.go +++ b/src/go/pt-k8s-debug-collector/main.go @@ -54,7 +54,7 @@ type cliOptions struct { func (c *cliOptions) AfterApply() error { if c.Version { fmt.Println(toolname) - fmt.Printf("Version %s\n", Version) + fmt.Printf("Version: %s\n", Version) fmt.Printf("Build: %s using %s\n", Build, GoVersion) fmt.Printf("Commit: %s\n", Commit) return nil From 7087bc360195010864b03737770568e37a81a94d Mon Sep 17 00:00:00 2001 From: Vladyslav Yurchenko Date: Fri, 3 Apr 2026 20:24:17 +0300 Subject: [PATCH 21/21] Fix folders export --- src/go/pt-k8s-debug-collector/dumper/individual_files.go | 6 ++++-- src/go/pt-k8s-debug-collector/main.go | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/go/pt-k8s-debug-collector/dumper/individual_files.go b/src/go/pt-k8s-debug-collector/dumper/individual_files.go index 3001a039d..e8bb9916e 100644 --- a/src/go/pt-k8s-debug-collector/dumper/individual_files.go +++ b/src/go/pt-k8s-debug-collector/dumper/individual_files.go @@ -103,7 +103,7 @@ func (d *Dumper) processSingleFile( dst := d.PodIndividualFilesPath( job.Pod.Namespace, job.Pod.Name, - path.Join(tarFolder, path.Base(filePath)), + path.Join(tarFolder, path.Clean(strings.TrimPrefix(filePath, "/"))), ) return d.archive.WriteFile(dst, tr, hdr.Size) @@ -124,6 +124,8 @@ func (d *Dumper) processDir( } defer rc.Close() + baseDir := path.Clean(strings.TrimPrefix(dir, "/")) + for { hdr, err := tr.Next() if err == io.EOF { @@ -154,7 +156,7 @@ func (d *Dumper) processDir( dst := d.PodIndividualFilesPath( job.Pod.Namespace, job.Pod.Name, - path.Join(tarFolder, path.Base(hdr.Name)), + path.Join(tarFolder, baseDir, relPath), ) if err := d.archive.WriteFile(dst, tr, hdr.Size); err != nil { diff --git a/src/go/pt-k8s-debug-collector/main.go b/src/go/pt-k8s-debug-collector/main.go index 3e86063ee..f31ccd5d6 100644 --- a/src/go/pt-k8s-debug-collector/main.go +++ b/src/go/pt-k8s-debug-collector/main.go @@ -54,7 +54,7 @@ type cliOptions struct { func (c *cliOptions) AfterApply() error { if c.Version { fmt.Println(toolname) - fmt.Printf("Version: %s\n", Version) + fmt.Printf("Version %s\n", Version) fmt.Printf("Build: %s using %s\n", Build, GoVersion) fmt.Printf("Commit: %s\n", Commit) return nil