diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 20a4b76..5e71777 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ jobs: - name: Install Go uses: buildjet/setup-go@v5 with: - go-version: 1.22.x + go-version: 1.25.x - name: golangci-lint uses: golangci/golangci-lint-action@v8 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1507a6..abb98a9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: - name: Setup Go uses: buildjet/setup-go@v5 with: - go-version: v1.22.x + go-version: v1.25.x - uses: buildjet/cache@v4 with: path: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36757dd..b77ea74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,7 @@ jobs: matrix: platform: [ubuntu-latest] go-version: - - 1.23.x - - 1.24.x + - 1.25.x runs-on: ${{ matrix.platform }} steps: - name: Install Go diff --git a/Makefile b/Makefile index a39c29f..635ab86 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,11 @@ test: .PHONY: lint lint: - golangci-lint run + mkdir -p bin + GOBIN=$(shell realpath bin) go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.4.0 + ./bin/golangci-lint run .PHONY: tidy tidy: - go mod tidy + go mod tidy diff --git a/collections/collections_suite_test.go b/collections/collections_suite_test.go new file mode 100644 index 0000000..61800c5 --- /dev/null +++ b/collections/collections_suite_test.go @@ -0,0 +1,13 @@ +package collections + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestCollections(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Collections Suite") +} diff --git a/collections/collections_test.go b/collections/collections_test.go index 7bf8f7b..aea9aac 100644 --- a/collections/collections_test.go +++ b/collections/collections_test.go @@ -1,121 +1,133 @@ package collections import ( - "reflect" - "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -func Test_MergeMap(t *testing.T) { - type args struct { - a map[string]string - b map[string]string - } - tests := []struct { - name string - args args - want map[string]string - }{ - { - name: "no overlaps", - args: args{ - a: map[string]string{"name": "flanksource"}, - b: map[string]string{"foo": "bar"}, - }, - want: map[string]string{ - "name": "flanksource", - "foo": "bar", - }, - }, - { - name: "overlaps", - args: args{ - a: map[string]string{"name": "flanksource", "foo": "baz"}, - b: map[string]string{"foo": "bar"}, - }, - want: map[string]string{ - "name": "flanksource", - "foo": "bar", - }, - }, - { - name: "overlaps II", - args: args{ - a: map[string]string{"name": "github", "foo": "baz"}, - b: map[string]string{"name": "flanksource", "foo": "bar"}, - }, - want: map[string]string{ - "name": "flanksource", - "foo": "bar", - }, - }, - { - name: "ditto", - args: args{ - a: map[string]string{"name": "flanksource", "foo": "bar"}, - b: map[string]string{"name": "flanksource", "foo": "bar"}, - }, - want: map[string]string{ - "name": "flanksource", - "foo": "bar", - }, - }, - { - name: "nil a", - args: args{ - a: nil, - b: map[string]string{"name": "flanksource", "foo": "bar"}, - }, - want: map[string]string{ - "name": "flanksource", - "foo": "bar", - }, - }, - { - name: "nil b", - args: args{ - a: map[string]string{"name": "flanksource", "foo": "bar"}, - b: nil, - }, - want: map[string]string{ - "name": "flanksource", - "foo": "bar", - }, - }, - { - name: "both nil", - args: args{ - a: nil, - b: nil, - }, - want: map[string]string{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := MergeMap(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { - t.Errorf("got %v, want %v", got, tt.want) - } - }) - } -} - -func Test_KeyValueSliceToMap(t *testing.T) { - tests := []struct { - name string - args []string - want map[string]string - }{ - {name: "simple", args: []string{"name=flanksource"}, want: map[string]string{"name": "flanksource"}}, - {name: "white space", args: []string{" name = flanksource "}, want: map[string]string{"name": "flanksource"}}, - {name: "multiple-simple", args: []string{"name=flanksource", "foo=bar"}, want: map[string]string{"name": "flanksource", "foo": "bar"}}, - {name: "double-equal", args: []string{"name=foo=bar"}, want: map[string]string{"name": "foo=bar"}}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := KeyValueSliceToMap(tt.args); !reflect.DeepEqual(got, tt.want) { - t.Errorf("got %v, want %v", got, tt.want) - } - }) - } -} +var _ = Describe("MergeMap", func() { + It("should merge maps with no overlaps", func() { + a := map[string]string{"name": "flanksource"} + b := map[string]string{"foo": "bar"} + expected := map[string]string{ + "name": "flanksource", + "foo": "bar", + } + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) + + It("should merge maps with overlaps, b takes precedence", func() { + a := map[string]string{"name": "flanksource", "foo": "baz"} + b := map[string]string{"foo": "bar"} + expected := map[string]string{ + "name": "flanksource", + "foo": "bar", + } + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) + + It("should merge maps with multiple overlaps", func() { + a := map[string]string{"name": "github", "foo": "baz"} + b := map[string]string{"name": "flanksource", "foo": "bar"} + expected := map[string]string{ + "name": "flanksource", + "foo": "bar", + } + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) + + It("should handle identical maps", func() { + a := map[string]string{"name": "flanksource", "foo": "bar"} + b := map[string]string{"name": "flanksource", "foo": "bar"} + expected := map[string]string{ + "name": "flanksource", + "foo": "bar", + } + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) + + It("should handle nil first map", func() { + var a map[string]string + b := map[string]string{"name": "flanksource", "foo": "bar"} + expected := map[string]string{ + "name": "flanksource", + "foo": "bar", + } + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) + + It("should handle nil second map", func() { + a := map[string]string{"name": "flanksource", "foo": "bar"} + var b map[string]string + expected := map[string]string{ + "name": "flanksource", + "foo": "bar", + } + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) + + It("should handle both maps nil", func() { + var a, b map[string]string + expected := map[string]string{} + + result := MergeMap(a, b) + + Expect(result).To(Equal(expected)) + }) +}) + +var _ = Describe("KeyValueSliceToMap", func() { + It("should convert simple key=value pair", func() { + args := []string{"name=flanksource"} + expected := map[string]string{"name": "flanksource"} + + result := KeyValueSliceToMap(args) + + Expect(result).To(Equal(expected)) + }) + + It("should handle whitespace around key=value pairs", func() { + args := []string{" name = flanksource "} + expected := map[string]string{"name": "flanksource"} + + result := KeyValueSliceToMap(args) + + Expect(result).To(Equal(expected)) + }) + + It("should convert multiple key=value pairs", func() { + args := []string{"name=flanksource", "foo=bar"} + expected := map[string]string{"name": "flanksource", "foo": "bar"} + + result := KeyValueSliceToMap(args) + + Expect(result).To(Equal(expected)) + }) + + It("should handle values with equal signs", func() { + args := []string{"name=foo=bar"} + expected := map[string]string{"name": "foo=bar"} + + result := KeyValueSliceToMap(args) + + Expect(result).To(Equal(expected)) + }) +}) diff --git a/collections/maps_test.go b/collections/maps_test.go index 07024fb..ba3ccd3 100644 --- a/collections/maps_test.go +++ b/collections/maps_test.go @@ -1,34 +1,20 @@ package collections -import "testing" +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) -func TestSortedMap(t *testing.T) { - type args struct { - labels map[string]string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "simple", - args: args{ - labels: map[string]string{ - "b": "b", - "a": "a", - "c": "c", - }, - }, - want: "a=a,b=b,c=c", - }, - } +var _ = Describe("SortedMap", func() { + It("should sort map entries alphabetically", func() { + labels := map[string]string{ + "b": "b", + "a": "a", + "c": "c", + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := SortedMap(tt.args.labels); got != tt.want { - t.Errorf("SortedMap() = %v, want %v", got, tt.want) - } - }) - } -} + result := SortedMap(labels) + + Expect(result).To(Equal("a=a,b=b,c=c")) + }) +}) diff --git a/collections/priorityqueue_test.go b/collections/priorityqueue_test.go index 28da758..e9a78b1 100644 --- a/collections/priorityqueue_test.go +++ b/collections/priorityqueue_test.go @@ -5,250 +5,267 @@ import ( "strings" "sync" "sync/atomic" - "testing" "time" "github.com/flanksource/commons/test/matchers" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -func TestPriorityQueueString(t *testing.T) { - g := NewWithT(t) - - pq, err := NewQueue(QueueOpts[string]{ - Comparator: strings.Compare, - Metrics: MetricsOpts[string]{ - Labels: map[string]any{ - "const_label": "value1", - }, - Labeller: map[string]func(i string) string{ - "prefix": func(i string) string { - if len(i) == 0 { - return i - } - return i[0:1] +var _ = Describe("PriorityQueue", func() { + Describe("String queue", func() { + It("should handle string priority queue operations", func() { + pq, err := NewQueue(QueueOpts[string]{ + Comparator: strings.Compare, + Metrics: MetricsOpts[string]{ + Labels: map[string]any{ + "const_label": "value1", + }, + Labeller: map[string]func(i string) string{ + "prefix": func(i string) string { + if len(i) == 0 { + return i + } + return i[0:1] + }, + }, }, - }, - }, - }) - - g.Expect(err).To(BeNil()) - g.Expect(pq.Size()).To(BeNumerically("==", 0)) - - pq.Enqueue("item1") - pq.Enqueue("batch1") + }) - g.Expect(pq.Size()).To(BeNumerically("==", 2)) - g.Expect(first(pq.Peek())).To(Equal("batch1")) - g.Expect(first(pq.Dequeue())).To(Equal("batch1")) - g.Expect(pq.Size()).To(BeNumerically("==", 1)) - g.Expect(first(pq.Dequeue())).Should(Equal("item1")) - g.Expect(first(pq.Peek())).To(Equal("")) + Expect(err).To(BeNil()) + Expect(pq.Size()).To(BeNumerically("==", 0)) - // g.Expect(dumpMetrics("priority")).Should(ContainSubstring("zz")) + pq.Enqueue("item1") + pq.Enqueue("batch1") - g.Expect("priority_queue_enqueued_total").To(matchers.MatchCounter(1, "prefix", "i")) - g.Expect("priority_queue_enqueued_total").To(matchers.MatchCounter(1, "prefix", "i")) - g.Expect("priority_queue_dequeued_total").To(matchers.MatchCounter(1, "prefix", "i")) - g.Expect("priority_queue_duration_count").To(matchers.MatchCounter(1, "prefix", "i")) - - g.Expect("priority_queue_size").To(matchers.MatchCounter(0)) -} - -type QueueItem struct { - Timestamp time.Time // Queued time - Obj map[string]any -} + Expect(pq.Size()).To(BeNumerically("==", 2)) + Expect(first(pq.Peek())).To(Equal("batch1")) + Expect(first(pq.Dequeue())).To(Equal("batch1")) + Expect(pq.Size()).To(BeNumerically("==", 1)) + Expect(first(pq.Dequeue())).Should(Equal("item1")) + Expect(first(pq.Peek())).To(Equal("")) -func (t *QueueItem) Name() string { - return t.Obj["name"].(string) -} + Expect("priority_queue_enqueued_total").To(matchers.MatchCounter(1, "prefix", "i")) + Expect("priority_queue_enqueued_total").To(matchers.MatchCounter(1, "prefix", "i")) + Expect("priority_queue_dequeued_total").To(matchers.MatchCounter(1, "prefix", "i")) + Expect("priority_queue_duration_count").To(matchers.MatchCounter(1, "prefix", "i")) -func NewQueueItem(obj map[string]any) *QueueItem { - return &QueueItem{ - Timestamp: time.Now(), - Obj: obj, - } -} - -func TestPriorityQueue(t *testing.T) { - g := NewWithT(t) - - pq, err := NewQueue(QueueOpts[*QueueItem]{ - Metrics: MetricsOpts[*QueueItem]{ - Labels: map[string]any{ - "const_label": "value1", - }, - Labeller: map[string]func(i *QueueItem) string{ - "prefix": func(i *QueueItem) string { - return "dummy" - }, - }, - }, - Comparator: func(a, b *QueueItem) int { - return strings.Compare(a.Obj["name"].(string), b.Obj["name"].(string)) - }, - Dedupe: true, - Equals: func(a, b *QueueItem) bool { - return strings.EqualFold(a.Obj["name"].(string), b.Obj["name"].(string)) - }, + Expect("priority_queue_size").To(matchers.MatchCounter(0)) + }) }) - g.Expect(err).To(BeNil()) - g.Expect(pq.Size()).To(BeZero()) - - names := []string{"bob", "foo", "bar", "eve", "baz", "alice", "bob"} - for _, name := range names { - pq.Enqueue(NewQueueItem(map[string]any{"name": name})) - } - g.Expect(pq.Size()).To(BeNumerically("==", len(names))) + Describe("Object queue", func() { + It("should handle object priority queue operations with deduplication", func() { + pq, err := NewQueue(QueueOpts[*QueueItem]{ + Metrics: MetricsOpts[*QueueItem]{ + Labels: map[string]any{ + "const_label": "value1", + }, + Labeller: map[string]func(i *QueueItem) string{ + "prefix": func(i *QueueItem) string { + return "dummy" + }, + }, + }, + Comparator: func(a, b *QueueItem) int { + return strings.Compare(a.Obj["name"].(string), b.Obj["name"].(string)) + }, + Dedupe: true, + Equals: func(a, b *QueueItem) bool { + return strings.EqualFold(a.Obj["name"].(string), b.Obj["name"].(string)) + }, + }) + Expect(err).To(BeNil()) + Expect(pq.Size()).To(BeZero()) - expected := []string{"alice", "bar", "baz", "bob", "eve", "foo"} - for _, e := range expected { - g.Expect(first(pq.Peek()).Name()).To(Equal(e)) - g.Expect(first(pq.Dequeue()).Name()).Should(Equal(e)) - } + names := []string{"bob", "foo", "bar", "eve", "baz", "alice", "bob"} + for _, name := range names { + pq.Enqueue(NewQueueItem(map[string]any{"name": name})) + } - g.Expect(pq.Size()).To(BeZero()) -} + Expect(pq.Size()).To(BeNumerically("==", len(names))) -func TestPriorityQueueDedupe(t *testing.T) { - g := NewWithT(t) + expected := []string{"alice", "bar", "baz", "bob", "eve", "foo"} + for _, e := range expected { + Expect(first(pq.Peek()).Name()).To(Equal(e)) + Expect(first(pq.Dequeue()).Name()).Should(Equal(e)) + } - pq, err := NewQueue(QueueOpts[string]{ - Equals: func(a, b string) bool { return a == b }, - Dedupe: true, - Comparator: strings.Compare, - Metrics: MetricsOpts[string]{ - Name: "dedupe_queue", - }, + Expect(pq.Size()).To(BeZero()) + }) }) - g.Expect(err).To(BeNil()) - g.Expect(pq.Size()).To(BeNumerically("==", 0)) + Describe("Deduplication", func() { + It("should handle deduplication correctly", func() { + pq, err := NewQueue(QueueOpts[string]{ + Equals: func(a, b string) bool { return a == b }, + Dedupe: true, + Comparator: strings.Compare, + Metrics: MetricsOpts[string]{ + Name: "dedupe_queue", + }, + }) - pq.Enqueue("item1") - pq.Enqueue("batch1") + Expect(err).To(BeNil()) + Expect(pq.Size()).To(BeNumerically("==", 0)) - g.Expect(pq.Size()).To(BeNumerically("==", 2)) - pq.Enqueue("item1") - pq.Enqueue("batch2") - g.Expect(pq.Size()).To(BeNumerically("==", 4)) + pq.Enqueue("item1") + pq.Enqueue("batch1") - pq.Dequeue() // batch1 - g.Expect(pq.Size()).To(BeNumerically("==", 3)) - pq.Dequeue() // batch2 - g.Expect(pq.Size()).To(BeNumerically("==", 2)) + Expect(pq.Size()).To(BeNumerically("==", 2)) + pq.Enqueue("item1") + pq.Enqueue("batch2") + Expect(pq.Size()).To(BeNumerically("==", 4)) - pq.Dequeue() // item1 - g.Expect(pq.Size()).To(BeNumerically("==", 0)) + pq.Dequeue() // batch1 + Expect(pq.Size()).To(BeNumerically("==", 3)) + pq.Dequeue() // batch2 + Expect(pq.Size()).To(BeNumerically("==", 2)) - g.Expect("dedupe_queue_enqueued_total").To(matchers.MatchCounter(4)) - g.Expect("dedupe_queue_dequeued_total").To(matchers.MatchCounter(3)) - g.Expect("dedupe_queue_deduped_total").To(matchers.MatchCounter(1)) - g.Expect("dedupe_queue_size").To(matchers.MatchCounter(0)) -} + pq.Dequeue() // item1 + Expect(pq.Size()).To(BeNumerically("==", 0)) -func TestPriorityQueueConcurrency(t *testing.T) { - g := NewWithT(t) - pq, err := NewQueue(QueueOpts[string]{ - Comparator: strings.Compare, - Metrics: MetricsOpts[string]{ - Name: "concurrent_queue", - }, + Expect("dedupe_queue_enqueued_total").To(matchers.MatchCounter(4)) + Expect("dedupe_queue_dequeued_total").To(matchers.MatchCounter(3)) + Expect("dedupe_queue_deduped_total").To(matchers.MatchCounter(1)) + Expect("dedupe_queue_size").To(matchers.MatchCounter(0)) + }) }) - g.Expect(err).To(BeNil()) - - const numGoroutines = 50 - const itemsPerGoroutine = 100 - - var wg sync.WaitGroup - start := time.Now() - // Start producer goroutines - for i := 0; i < numGoroutines; i++ { - wg.Add(1) - go func(id int) { - defer wg.Done() - for j := 0; j < itemsPerGoroutine; j++ { - pq.Enqueue(fmt.Sprintf("item-%d-%d", id, j)) + Describe("Concurrency", func() { + It("should handle concurrent operations safely", func() { + pq, err := NewQueue(QueueOpts[string]{ + Comparator: strings.Compare, + Metrics: MetricsOpts[string]{ + Name: "concurrent_queue", + }, + }) + Expect(err).To(BeNil()) + + const numGoroutines = 50 + const itemsPerGoroutine = 100 + + var wg sync.WaitGroup + + start := time.Now() + // Start producer goroutines + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + for j := 0; j < itemsPerGoroutine; j++ { + pq.Enqueue(fmt.Sprintf("item-%d-%d", id, j)) + } + }(i) } - }(i) - } - // wg.Wait() - // g.Expect(time.Since(start).Milliseconds()).To(BeNumerically("<", 100)) - // g.Expect(pq.Size()).To(BeNumerically("==", itemsPerGoroutine*numGoroutines)) - // start = time.Now() - // Start consumer goroutines - expectedCount := numGoroutines * itemsPerGoroutine - dequeued := atomic.Int32{} - for i := 0; i < numGoroutines; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for dequeued.Load() < int32(expectedCount) { - if _, ok := pq.Dequeue(); ok { - dequeued.Add(1) - } else { - time.Sleep(1 * time.Millisecond) - } + // Start consumer goroutines + expectedCount := numGoroutines * itemsPerGoroutine + dequeued := atomic.Int32{} + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for dequeued.Load() < int32(expectedCount) { + if _, ok := pq.Dequeue(); ok { + dequeued.Add(1) + } else { + time.Sleep(1 * time.Millisecond) + } + } + }() } - }() - } - // Wait for all operations to complete - wg.Wait() - g.Expect(time.Since(start).Milliseconds()).To(BeNumerically("<", 100)) - g.Expect(int(dequeued.Load())).To(Equal(expectedCount)) - g.Expect("concurrent_queue_duration_count").To(matchers.MatchCounter(int64(expectedCount))) + // Wait for all operations to complete + wg.Wait() + Expect(time.Since(start).Milliseconds()).To(BeNumerically("<", 100)) + Expect(int(dequeued.Load())).To(Equal(expectedCount)) + Expect("concurrent_queue_duration_count").To(matchers.MatchCounter(int64(expectedCount))) - g.Expect("concurrent_queue_size").To(matchers.MatchCounter(0)) - g.Expect(pq.Size()).To(BeNumerically("==", 0)) + Expect("concurrent_queue_size").To(matchers.MatchCounter(0)) + Expect(pq.Size()).To(BeNumerically("==", 0)) - t.Log("\n" + matchers.DumpMetrics("priority")) -} + GinkgoWriter.Printf("\n" + matchers.DumpMetrics("priority")) + }) + }) -func TestPriorityQueueWithDelay(t *testing.T) { - g := NewWithT(t) + Describe("Delayed enqueuing", func() { + It("should handle delayed items correctly", func() { + pq, err := NewQueue(QueueOpts[string]{ + Equals: func(a, b string) bool { return a == b }, + Comparator: strings.Compare, + Metrics: MetricsOpts[string]{ + Disable: true, + }, + }) - pq, err := NewQueue(QueueOpts[string]{ - Equals: func(a, b string) bool { return a == b }, - Comparator: strings.Compare, - Metrics: MetricsOpts[string]{ - Disable: true, - }, - }) + Expect(err).To(BeNil()) + Expect(pq.Size()).To(BeNumerically("==", 0)) - g.Expect(err).To(BeNil()) - g.Expect(pq.Size()).To(BeNumerically("==", 0)) + // Enqueue items with different delays: 3s, 500ms, and immediate + pq.EnqueueWithDelay("item1-delay-3s", 3*time.Second) + pq.EnqueueWithDelay("item1-delay-500ms", 500*time.Millisecond) + pq.Enqueue("item1-immediate") - pq.EnqueueWithDelay("item1-delay-06s", 6*time.Second) - pq.EnqueueWithDelay("item1-delay-03s", 3*time.Second) - pq.Enqueue("item1") + Expect(pq.Size()).To(BeNumerically("==", 3)) - start := time.Now() + // Immediate item should be available right away + item, ok := pq.Dequeue() + Expect(ok).To(BeTrue()) + Expect(item).To(Equal("item1-immediate")) - var items []string - for pq.Size() != 0 { + // 500ms delayed item should not be available immediately + item, ok = pq.Dequeue() + Expect(ok).To(BeFalse()) + Expect(item).To(BeEmpty()) - item, valid := pq.Dequeue() - if valid { - items = append(items, item) - } + // After 600ms, the 500ms delayed item should be available + Eventually(func() string { + item, ok := pq.Dequeue() + if !ok { + return "" + } + return item + }, 800*time.Millisecond, 50*time.Millisecond).Should(Equal("item1-delay-500ms")) + + // 3s delayed item should not be available yet + Consistently(func() bool { + _, ok := pq.Dequeue() + return ok + }, 1*time.Second, 100*time.Millisecond).Should(BeFalse()) + + // After 3.5s total, the 3s delayed item should be available + Eventually(func() string { + item, ok := pq.Dequeue() + if !ok { + return "" + } + return item + }, 3*time.Second, 100*time.Millisecond).Should(Equal("item1-delay-3s")) + + // Queue should now be empty + Expect(pq.Size()).To(BeZero()) + item, ok = pq.Dequeue() + Expect(ok).To(BeFalse()) + Expect(item).To(BeEmpty()) + }) + }) +}) - if time.Since(start) > (1*time.Second) && time.Since(start) < (2*time.Second) { - g.Expect(items).To(Equal([]string{"item1"})) - } - if time.Since(start) > (3*time.Second) && time.Since(start) < (5*time.Second) { - g.Expect(items).To(Equal([]string{"item1", "item1-delay-03s"})) - } - if time.Since(start) > (6 * time.Second) { - g.Expect(items).To(Equal([]string{"item1", "item1-delay-03s", "item1-delay-06s"})) - } - } +type QueueItem struct { + Timestamp time.Time // Queued time + Obj map[string]any +} - g.Expect(items).To(Equal([]string{"item1", "item1-delay-03s", "item1-delay-06s"})) +func (t *QueueItem) Name() string { + return t.Obj["name"].(string) +} + +func NewQueueItem(obj map[string]any) *QueueItem { + return &QueueItem{ + Timestamp: time.Now(), + Obj: obj, + } } func first[T1 any, T2 any](a T1, _ T2) T1 { diff --git a/collections/set/set_suite_test.go b/collections/set/set_suite_test.go new file mode 100644 index 0000000..e19482f --- /dev/null +++ b/collections/set/set_suite_test.go @@ -0,0 +1,13 @@ +package set + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSet(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Set Suite") +} \ No newline at end of file diff --git a/collections/set/set_test.go b/collections/set/set_test.go index bce3ee0..8b3c1eb 100644 --- a/collections/set/set_test.go +++ b/collections/set/set_test.go @@ -2,70 +2,101 @@ package set import ( "encoding/json" - "testing" - "github.com/stretchr/testify/assert" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -func Test_StringSet(t *testing.T) { - s := New("a", "b", "c", "d") - s.Add("e") +var _ = Describe("Set", func() { + Describe("String Set", func() { + It("should handle basic string set operations", func() { + s := New("a", "b", "c", "d") + s.Add("e") - assert.ElementsMatch(t, s.ToSlice(), []string{"a", "b", "c", "d", "e"}) + Expect(s.ToSlice()).To(ConsistOf("a", "b", "c", "d", "e")) - s.Add("a") - assert.ElementsMatch(t, s.ToSlice(), []string{"a", "b", "c", "d", "e"}) + s.Add("a") // Adding duplicate + Expect(s.ToSlice()).To(ConsistOf("a", "b", "c", "d", "e")) - s.Remove("b") - s.Remove("c") - assert.ElementsMatch(t, s.ToSlice(), []string{"a", "d", "e"}) + s.Remove("b") + s.Remove("c") + Expect(s.ToSlice()).To(ConsistOf("a", "d", "e")) - assert.Equal(t, s.Contains("a"), true) - assert.Equal(t, s.Contains("z"), false) + Expect(s.Contains("a")).To(BeTrue()) + Expect(s.Contains("z")).To(BeFalse()) + }) - s2 := New("d", "e", "f", "g", "h") - assert.ElementsMatch(t, s.Union(s2).ToSlice(), []string{"a", "d", "e", "f", "g", "h"}) + It("should handle union operations", func() { + s := New("a", "d", "e") + s2 := New("d", "e", "f", "g", "h") - assert.ElementsMatch(t, s.Intersection(s2).ToSlice(), []string{"d", "e"}) -} + result := s.Union(s2) + Expect(result.ToSlice()).To(ConsistOf("a", "d", "e", "f", "g", "h")) + }) -func Test_IntSet(t *testing.T) { - s := New(1, 2, 3, 4) - s.Add(5) + It("should handle intersection operations", func() { + s := New("a", "d", "e") + s2 := New("d", "e", "f", "g", "h") - assert.ElementsMatch(t, s.ToSlice(), []int{1, 2, 3, 4, 5}) + result := s.Intersection(s2) + Expect(result.ToSlice()).To(ConsistOf("d", "e")) + }) + }) - s.Add(1) - assert.ElementsMatch(t, s.ToSlice(), []int{1, 2, 3, 4, 5}) + Describe("Integer Set", func() { + It("should handle basic integer set operations", func() { + s := New(1, 2, 3, 4) + s.Add(5) - s.Remove(2) - s.Remove(3) - assert.ElementsMatch(t, s.ToSlice(), []int{1, 4, 5}) + Expect(s.ToSlice()).To(ConsistOf(1, 2, 3, 4, 5)) - assert.Equal(t, s.Contains(1), true) - assert.Equal(t, s.Contains(100), false) + s.Add(1) // Adding duplicate + Expect(s.ToSlice()).To(ConsistOf(1, 2, 3, 4, 5)) - s2 := New(4, 5, 6, 7, 8) - assert.ElementsMatch(t, s.Union(s2).ToSlice(), []int{1, 4, 5, 6, 7, 8}) + s.Remove(2) + s.Remove(3) + Expect(s.ToSlice()).To(ConsistOf(1, 4, 5)) - assert.ElementsMatch(t, s.Intersection(s2).ToSlice(), []int{4, 5}) -} + Expect(s.Contains(1)).To(BeTrue()) + Expect(s.Contains(100)).To(BeFalse()) + }) -func Test_JSON(t *testing.T) { - type Fruits struct { - Names Set[string] `json:"names"` - } + It("should handle union operations", func() { + s := New(1, 4, 5) + s2 := New(4, 5, 6, 7, 8) - f := Fruits{Names: New("orange", "apple", "orange", "banana", "mango")} + result := s.Union(s2) + Expect(result.ToSlice()).To(ConsistOf(1, 4, 5, 6, 7, 8)) + }) - b, err := json.Marshal(f) - assert.NoError(t, err) + It("should handle intersection operations", func() { + s := New(1, 4, 5) + s2 := New(4, 5, 6, 7, 8) - assert.Equal(t, len(`{"names":["banana","mango","orange","apple"]}`), len(string(b))) + result := s.Intersection(s2) + Expect(result.ToSlice()).To(ConsistOf(4, 5)) + }) + }) - var jsonFruits Fruits - err = json.Unmarshal(b, &jsonFruits) - assert.NoError(t, err) + Describe("JSON Serialization", func() { + It("should marshal and unmarshal sets correctly", func() { + type Fruits struct { + Names Set[string] `json:"names"` + } - assert.ElementsMatch(t, f.Names.ToSlice(), jsonFruits.Names.ToSlice()) -} + f := Fruits{Names: New("orange", "apple", "orange", "banana", "mango")} + + b, err := json.Marshal(f) + Expect(err).ToNot(HaveOccurred()) + + // Check that the JSON has the expected length (order may vary) + Expect(len(string(b))).To(Equal(len(`{"names":["banana","mango","orange","apple"]}`))) + + var jsonFruits Fruits + err = json.Unmarshal(b, &jsonFruits) + Expect(err).ToNot(HaveOccurred()) + + Expect(jsonFruits.Names.ToSlice()).To(ConsistOf(f.Names.ToSlice())) + }) + }) +}) diff --git a/collections/slice_test.go b/collections/slice_test.go index 31bc582..7bbd6f3 100644 --- a/collections/slice_test.go +++ b/collections/slice_test.go @@ -1,210 +1,65 @@ package collections import ( - "reflect" - "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -func TestMatchItems(t *testing.T) { - tests := []struct { - name string - item string - patterns []string - expected bool - }{ - { - name: "Exact Match", - item: "apple", - patterns: []string{"apple"}, - expected: true, - }, - { - name: "Negative Match", - item: "apple", - patterns: []string{"!apple"}, - expected: false, - }, - { - name: "Empty Items List", - item: "apple", - patterns: []string{}, - expected: true, - }, - { - name: "Wildcard Match", - item: "apple", - patterns: []string{"*"}, - expected: true, - }, - { - name: "Wildcard Prefix Match", - item: "apple", - patterns: []string{"appl*"}, - expected: true, - }, - { - name: "Wildcard Suffix Match", - item: "apple", - patterns: []string{"*ple"}, - expected: true, - }, - { - name: "Mixed Matches", - item: "apple", - patterns: []string{"!banana", "appl*", "cherry"}, - expected: true, - }, - { - name: "No Items Match", - item: "apple", - patterns: []string{"!apple", "banana"}, - expected: false, - }, - { - name: "Multiple Wildcards", - item: "apple", - patterns: []string{"ap*e", "*pl*e"}, - expected: false, - }, - { - name: "Glob", - item: "apple", - patterns: []string{"*ppl*"}, - expected: true, - }, - { - name: "Handle whitespaces | should be trimmed", - item: "hello", - patterns: []string{"hello ", "world"}, - expected: true, - }, - { - name: "Handle whitespaces | should not be trimmed (no match)", - item: "hello", - patterns: []string{"hello%20", "world"}, - expected: false, - }, - { - name: "Handle whitespaces | should not be trimmed (match)", - item: "hello ", - patterns: []string{"hello%20", "world"}, - expected: true, - }, - { - name: "exclusion and inclusion", - item: "mission-control", - patterns: []string{"!mission-control", "mission-control"}, - expected: false, - }, - { - name: "inclusion and exclusion", - item: "mission-control", - patterns: []string{"mission-control", "!mission-control"}, - expected: false, - }, - { - name: "exclusion", - item: "mission-control", - patterns: []string{"!default"}, - expected: true, - }, - { - name: "Exclude All", - item: "anyitem", - patterns: []string{"!*"}, - expected: false, - }, - { - name: "Exclude All with Inclusion", - item: "apple", - patterns: []string{"!*", "apple"}, - expected: false, - }, - { - name: "Multiple Exclusions", - item: "apple", - patterns: []string{"!banana", "!orange", "!apple"}, - expected: false, - }, - { - name: "Empty Item with Patterns", - item: "", - patterns: []string{"*"}, - expected: true, - }, - { - name: "Empty Pattern String", - item: "apple", - patterns: []string{""}, - expected: false, - }, - { - name: "URL Encoded Pattern Matches", - item: "hello ", - patterns: []string{"hello%20"}, - expected: true, - }, - { - name: "URL Encoded Pattern Does Not Match", - item: "hello", - patterns: []string{"hello%20"}, - expected: false, - }, - { - name: "Malformed URL Encoding", - item: "apple", - patterns: []string{"%zzapple"}, - expected: false, - }, - } +var _ = Describe("MatchItems", func() { + DescribeTable("pattern matching scenarios", + func(item string, patterns []string, expected bool) { + result := MatchItems(item, patterns...) + Expect(result).To(Equal(expected)) + }, + Entry("exact match", "apple", []string{"apple"}, true), + Entry("negative match", "apple", []string{"!apple"}, false), + Entry("empty items list", "apple", []string{}, true), + Entry("wildcard match", "apple", []string{"*"}, true), + Entry("wildcard prefix match", "apple", []string{"appl*"}, true), + Entry("wildcard suffix match", "apple", []string{"*ple"}, true), + Entry("mixed matches", "apple", []string{"!banana", "appl*", "cherry"}, true), + Entry("no items match", "apple", []string{"!apple", "banana"}, false), + Entry("multiple wildcards", "apple", []string{"ap*e", "*pl*e"}, false), + Entry("glob", "apple", []string{"*ppl*"}, true), + Entry("handle whitespaces - should be trimmed", "hello", []string{"hello ", "world"}, true), + Entry("handle whitespaces - should not be trimmed (no match)", "hello", []string{"hello%20", "world"}, false), + Entry("handle whitespaces - should not be trimmed (match)", "hello ", []string{"hello%20", "world"}, true), + Entry("exclusion and inclusion", "mission-control", []string{"!mission-control", "mission-control"}, false), + Entry("inclusion and exclusion", "mission-control", []string{"mission-control", "!mission-control"}, false), + Entry("exclusion", "mission-control", []string{"!default"}, true), + Entry("exclude all", "anyitem", []string{"!*"}, false), + Entry("exclude all with inclusion", "apple", []string{"!*", "apple"}, false), + Entry("multiple exclusions", "apple", []string{"!banana", "!orange", "!apple"}, false), + Entry("empty item with patterns", "", []string{"*"}, true), + Entry("empty pattern string", "apple", []string{""}, false), + Entry("URL encoded pattern matches", "hello ", []string{"hello%20"}, true), + Entry("URL encoded pattern does not match", "hello", []string{"hello%20"}, false), + Entry("malformed URL encoding", "apple", []string{"%zzapple"}, false), + ) +}) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := MatchItems(test.item, test.patterns...) - if result != test.expected { - t.Errorf("Expected %v but got %v", test.expected, result) - } - }) - } -} +var _ = Describe("Append", func() { + It("should append string slices", func() { + slices := [][]any{ + {"a", "b"}, + {"c"}, + {"d", "e", "f"}, + } -func TestAppend(t *testing.T) { - type args struct { - slices [][]any - } - tests := []struct { - name string - args args - want []any - }{ - { - name: "strings", - args: args{ - slices: [][]any{ - {"a", "b"}, - {"c"}, - {"d", "e", "f"}, - }, - }, - want: []any{"a", "b", "c", "d", "e", "f"}, - }, - { - name: "ints", - args: args{ - slices: [][]any{ - {1, 2, 3}, - {4, 5, 6}, - {7, 8, 9}, - }, - }, - want: []any{1, 2, 3, 4, 5, 6, 7, 8, 9}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := Append(tt.args.slices...); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Append() = %v, want %v", got, tt.want) - } - }) - } -} + result := Append(slices...) + + Expect(result).To(Equal([]any{"a", "b", "c", "d", "e", "f"})) + }) + + It("should append integer slices", func() { + slices := [][]any{ + {1, 2, 3}, + {4, 5, 6}, + {7, 8, 9}, + } + + result := Append(slices...) + + Expect(result).To(Equal([]any{1, 2, 3, 4, 5, 6, 7, 8, 9})) + }) +}) diff --git a/console/test_results.go b/console/test_results.go index c358e1d..d32df0f 100644 --- a/console/test_results.go +++ b/console/test_results.go @@ -137,7 +137,7 @@ func (c *TestResults) Tracef(s string, args ...interface{}) { func (c *TestResults) Assert(name string, expect, actual interface{}) { if expect == actual { - c.Passf(name, name) + c.Passf(name, "%s", name) } else { c.Failf(name, "expected \"%v\", got \"%vs\"", expect, actual) } diff --git a/files/archive_test.go b/files/archive_test.go new file mode 100644 index 0000000..ee32f5b --- /dev/null +++ b/files/archive_test.go @@ -0,0 +1,522 @@ +package files + +import ( + "os" + "path/filepath" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestArchive(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Archive Suite") +} + +type testCase struct { + name string + archivePath string + expectedArchive *Archive + expectedError string +} + +var _ = Describe("Unarchive", func() { + testCases := []testCase{ + { + name: "simple tar archive", + archivePath: "testdata/archives/simple.tar", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt"}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "simple gzipped tar archive", + archivePath: "testdata/archives/simple.tar.gz", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt"}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "simple xz compressed tar archive", + archivePath: "testdata/archives/simple.tar.xz", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt"}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "nested tar with directories", + archivePath: "testdata/archives/nested.tar", + expectedArchive: &Archive{ + Files: []string{"./file1.txt", "./file2.txt", "./subdir/file3.txt"}, + Directories: []string{"./", "./subdir/"}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "tar with safe symlinks", + archivePath: "testdata/archives/safe_symlinks.tar", + expectedArchive: &Archive{ + Files: []string{"target.txt"}, + Directories: []string{}, + Symlinks: []string{"safe_link"}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "tar with dangerous symlinks should fail", + archivePath: "testdata/archives/parent_symlinks.tar", + expectedError: "is absolute and not allowed", + }, + { + name: "simple zip archive", + archivePath: "testdata/archives/simple.zip", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt"}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "jar archive", + archivePath: "testdata/archives/test.jar", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt"}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "tgz archive", + archivePath: "testdata/archives/nested.tgz", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt", "subdir/file3.txt"}, + Directories: []string{"subdir/"}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "txz archive", + archivePath: "testdata/archives/nested.txz", + expectedArchive: &Archive{ + Files: []string{"file1.txt", "file2.txt", "subdir/file3.txt"}, + Directories: []string{"subdir/"}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "tar.xz with safe symlinks", + archivePath: "testdata/archives/safe_postgres.tar.xz", + expectedArchive: &Archive{ + Files: []string{"./bin/postgres", "./bin/postgres.conf"}, + Directories: []string{"./", "./bin/"}, + Symlinks: []string{"./bin/config_link"}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "tar.xz with parent symlinks should fail", + archivePath: "testdata/archives/symlinks.tar.xz", + expectedError: "is absolute and not allowed", + }, + { + name: "empty archive", + archivePath: "testdata/archives/empty.tar", + expectedArchive: &Archive{ + Files: []string{}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "dirs only archive", + archivePath: "testdata/archives/dirs_only.tar", + expectedArchive: &Archive{ + Files: []string{}, + Directories: []string{"./", "./subdir1/", "./subdir1/subdir2/", "./subdir3/"}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "hard links in tar", + archivePath: "testdata/archives/hardlinks.tar", + expectedArchive: &Archive{ + Files: []string{"./hardlink.txt", "./original.txt"}, + Directories: []string{"./"}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "special characters in zip", + archivePath: "testdata/archives/special_chars.zip", + expectedArchive: &Archive{ + Files: []string{"special_chars/file with spaces.txt", "special_chars/file_with_underscore.txt"}, + Directories: []string{}, + Symlinks: []string{}, + Skipped: []string{}, + Errors: []error{}, + }, + }, + { + name: "corrupted archive", + archivePath: "testdata/archives/corrupted.tar.gz", + expectedError: "unexpected EOF", + }, + { + name: "unsupported format", + archivePath: "testdata/archives/nonexistent.rar", + expectedError: "unknown format type", + }, + } + + for _, tc := range testCases { + tc := tc // capture range variable + It(tc.name, func() { + // Create temporary directory for extraction + tempDir, err := os.MkdirTemp("", "unarchive-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // Run the extraction + archive, err := Unarchive(tc.archivePath, tempDir) + + if tc.expectedError != "" { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(tc.expectedError)) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(archive).NotTo(BeNil()) + + // Verify extracted files match expectations + Expect(archive.Files).To(ConsistOf(tc.expectedArchive.Files)) + Expect(archive.Directories).To(ConsistOf(tc.expectedArchive.Directories)) + Expect(archive.Symlinks).To(ConsistOf(tc.expectedArchive.Symlinks)) + + // Verify source and destination paths are set + absArchivePath, _ := filepath.Abs(tc.archivePath) + absTempDir, _ := filepath.Abs(tempDir) + Expect(archive.Source).To(Equal(absArchivePath)) + Expect(archive.Destination).To(Equal(absTempDir)) + + // Verify extracted size is greater than 0 for non-empty archives + if len(tc.expectedArchive.Files) > 0 { + Expect(archive.ExtractedSize).To(BeNumerically(">", 0)) + } + + // Verify compressed size is greater than 0 + Expect(archive.CompressedSize).To(BeNumerically(">", 0)) + + // Verify actual files exist on disk + for _, fileName := range tc.expectedArchive.Files { + filePath := filepath.Join(tempDir, fileName) + Expect(filePath).To(BeAnExistingFile()) + } + + // Verify directories exist on disk + for _, dirName := range tc.expectedArchive.Directories { + dirPath := filepath.Join(tempDir, dirName) + Expect(dirPath).To(BeADirectory()) + } + + // Verify symlinks exist on disk + for _, linkName := range tc.expectedArchive.Symlinks { + linkPath := filepath.Join(tempDir, linkName) + info, err := os.Lstat(linkPath) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Mode() & os.ModeSymlink).NotTo(BeZero()) + } + } + }) + } +}) + +var _ = Describe("UntarWithFilter", func() { + It("should extract only executable files", func() { + tempDir, err := os.MkdirTemp("", "filter-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // Test executable filter + err = UntarWithFilter("testdata/archives/executable.tar", tempDir, func(header os.FileInfo) string { + if header.Mode()&0111 != 0 { // Has execute permission + return header.Name() + } + return "" // Skip non-executable files + }) + + Expect(err).NotTo(HaveOccurred()) + + // Verify only executable was extracted + scriptPath := filepath.Join(tempDir, "script.sh") + regularPath := filepath.Join(tempDir, "regular.txt") + + Expect(scriptPath).To(BeAnExistingFile()) + Expect(regularPath).NotTo(BeAnExistingFile()) + }) + + It("should apply custom name filter", func() { + tempDir, err := os.MkdirTemp("", "filter-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // Test custom filter - only files containing "file1" + err = UntarWithFilter("testdata/archives/simple.tar", tempDir, func(header os.FileInfo) string { + if strings.Contains(header.Name(), "file1") { + return header.Name() + } + return "" + }) + + Expect(err).NotTo(HaveOccurred()) + + // Verify only file1 was extracted + file1Path := filepath.Join(tempDir, "file1.txt") + file2Path := filepath.Join(tempDir, "file2.txt") + + Expect(file1Path).To(BeAnExistingFile()) + Expect(file2Path).NotTo(BeAnExistingFile()) + }) +}) + +var _ = Describe("UnarchiveExecutables", func() { + It("should extract only executable files using the built-in filter", func() { + tempDir, err := os.MkdirTemp("", "executable-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + err = UnarchiveExecutables("testdata/archives/executable.tar", tempDir) + Expect(err).NotTo(HaveOccurred()) + + // Verify only executable was extracted to base name + scriptPath := filepath.Join(tempDir, "script.sh") + regularPath := filepath.Join(tempDir, "regular.txt") + + Expect(scriptPath).To(BeAnExistingFile()) + Expect(regularPath).NotTo(BeAnExistingFile()) + }) +}) + +var _ = Describe("Unarchive with Overwrite Options", func() { + type overwriteTestCase struct { + name string + archivePath string + overwriteOption bool + expectedError string + } + + overwriteTestCases := []overwriteTestCase{ + { + name: "no overwrite with existing files should fail", + archivePath: "testdata/archives/simple.tar", + overwriteOption: false, + expectedError: "file file1.txt already exists", + }, + { + name: "with overwrite should succeed", + archivePath: "testdata/archives/simple.tar", + overwriteOption: true, + expectedError: "", + }, + { + name: "no overwrite with zip and existing files should fail", + archivePath: "testdata/archives/simple.zip", + overwriteOption: false, + expectedError: "file file1.txt already exists", + }, + { + name: "with overwrite for zip should succeed", + archivePath: "testdata/archives/simple.zip", + overwriteOption: true, + expectedError: "", + }, + } + + for _, tc := range overwriteTestCases { + tc := tc // capture range variable + It(tc.name, func() { + // Create temporary directory for extraction + tempDir, err := os.MkdirTemp("", "overwrite-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // First extraction - should always succeed + _, err = Unarchive(tc.archivePath, tempDir) + Expect(err).NotTo(HaveOccurred()) + + // Create different content in existing files to verify overwrite behavior + existingFiles := []string{"file1.txt", "file2.txt"} + originalContent := "ORIGINAL CONTENT - SHOULD BE REPLACED" + for _, fileName := range existingFiles { + filePath := filepath.Join(tempDir, fileName) + err := os.WriteFile(filePath, []byte(originalContent), 0644) + Expect(err).NotTo(HaveOccurred()) + } + + // Second extraction - behavior depends on overwrite option + var archive *Archive + if tc.overwriteOption { + archive, err = Unarchive(tc.archivePath, tempDir, WithOverwrite(true)) + } else { + archive, err = Unarchive(tc.archivePath, tempDir) + } + + if tc.expectedError != "" { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal(tc.expectedError)) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(archive).NotTo(BeNil()) + + // Verify Overwritten list contains the expected files in exact order + Expect(archive.Overwritten).To(Equal(existingFiles)) + + // Verify files were actually overwritten (content changed) + for _, fileName := range existingFiles { + filePath := filepath.Join(tempDir, fileName) + content, err := os.ReadFile(filePath) + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).NotTo(ContainSubstring("ORIGINAL CONTENT")) + } + } + }) + } + + It("should track mixed overwrite scenario correctly", func() { + tempDir, err := os.MkdirTemp("", "mixed-overwrite-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // Create only some files that will be overwritten + existingFile := filepath.Join(tempDir, "file1.txt") + err = os.WriteFile(existingFile, []byte("existing content"), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Extract archive with overwrite enabled + archive, err := Unarchive("testdata/archives/simple.tar", tempDir, WithOverwrite(true)) + Expect(err).NotTo(HaveOccurred()) + + // Should only list the one file that was overwritten + Expect(archive.Overwritten).To(Equal([]string{"file1.txt"})) + + // Should still list both extracted files + Expect(archive.Files).To(ConsistOf([]string{"file1.txt", "file2.txt"})) + }) + + It("should handle hard links overwrite correctly", func() { + tempDir, err := os.MkdirTemp("", "hardlink-overwrite-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // First extraction + _, err = Unarchive("testdata/archives/hardlinks.tar", tempDir) + Expect(err).NotTo(HaveOccurred()) + + // Modify existing files + files := []string{"./original.txt", "./hardlink.txt"} + for _, fileName := range files { + filePath := filepath.Join(tempDir, fileName) + err := os.WriteFile(filePath, []byte("modified content"), 0644) + Expect(err).NotTo(HaveOccurred()) + } + + // Second extraction with overwrite + archive, err := Unarchive("testdata/archives/hardlinks.tar", tempDir, WithOverwrite(true)) + Expect(err).NotTo(HaveOccurred()) + + // Should track both overwritten files with exact paths + Expect(archive.Overwritten).To(Equal(files)) + }) + + It("should verify error message contains the first conflicting file", func() { + tempDir, err := os.MkdirTemp("", "error-message-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // Create only the first file that would conflict + existingFile := filepath.Join(tempDir, "file1.txt") + err = os.WriteFile(existingFile, []byte("existing content"), 0644) + Expect(err).NotTo(HaveOccurred()) + + // Extract without overwrite should fail with specific filename + _, err = Unarchive("testdata/archives/simple.tar", tempDir) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("file file1.txt already exists")) + }) + + It("should preserve path format from archive in Overwritten list", func() { + tempDir, err := os.MkdirTemp("", "path-format-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // First extraction + _, err = Unarchive("testdata/archives/nested.tar", tempDir) + Expect(err).NotTo(HaveOccurred()) + + // Modify existing files (using the exact paths from the archive in correct order) + existingFiles := []string{"./file2.txt", "./file1.txt", "./subdir/file3.txt"} + for _, fileName := range existingFiles { + filePath := filepath.Join(tempDir, fileName) + err := os.WriteFile(filePath, []byte("modified content"), 0644) + Expect(err).NotTo(HaveOccurred()) + } + + // Second extraction with overwrite + archive, err := Unarchive("testdata/archives/nested.tar", tempDir, WithOverwrite(true)) + Expect(err).NotTo(HaveOccurred()) + + // Should preserve the exact path format from the archive + Expect(archive.Overwritten).To(Equal(existingFiles)) + }) + + It("should maintain overwrite order matching extraction order", func() { + tempDir, err := os.MkdirTemp("", "order-test-") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tempDir) + + // Create files in reverse order to test that overwrite order follows extraction order + existingFiles := []string{"file2.txt", "file1.txt"} + for _, fileName := range existingFiles { + filePath := filepath.Join(tempDir, fileName) + err := os.WriteFile(filePath, []byte("existing content"), 0644) + Expect(err).NotTo(HaveOccurred()) + } + + // Extract with overwrite + archive, err := Unarchive("testdata/archives/simple.tar", tempDir, WithOverwrite(true)) + Expect(err).NotTo(HaveOccurred()) + + // Overwritten should follow the order they appear in the archive, not filesystem order + Expect(archive.Overwritten).To(Equal([]string{"file1.txt", "file2.txt"})) + }) +}) diff --git a/files/files.go b/files/files.go index c0fe85f..3288cee 100644 --- a/files/files.go +++ b/files/files.go @@ -22,6 +22,105 @@ import ( "github.com/ulikunitz/xz" ) +// Archive represents the result of an archive extraction operation +type Archive struct { + Source string // Path to source archive + Destination string // Extraction destination + Files []string // Successfully extracted files + Directories []string // Created directories + Symlinks []string // Created symlinks + Skipped []string // Skipped entries + Errors []error // Non-fatal errors encountered + Overwritten []string // Files that were overwritten during extraction + + // Size metrics + CompressedSize int64 // Size of the archive file + ExtractedSize int64 // Total size of extracted content + CompressionRatio float64 // Ratio of extracted to compressed size +} + +// UnarchiveOptions configures archive extraction behavior +type UnarchiveOptions struct { + Overwrite bool // Allow overwriting existing files +} + +// UnarchiveOption is a functional option for configuring archive extraction +type UnarchiveOption func(*UnarchiveOptions) + +// WithOverwrite sets whether existing files should be overwritten during extraction +func WithOverwrite(overwrite bool) UnarchiveOption { + return func(opts *UnarchiveOptions) { + opts.Overwrite = overwrite + } +} + +// formatBytes converts bytes to human readable format +func formatBytes(bytes int64) string { + const unit = 1024 + if bytes < unit { + return fmt.Sprintf("%d B", bytes) + } + div, exp := int64(unit), 0 + for n := bytes / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) +} + +// String returns a human-readable summary of the archive extraction +func (a *Archive) String() string { + var parts []string + + // Main extraction info + if len(a.Files) > 0 || a.ExtractedSize > 0 { + if a.ExtractedSize > 0 { + parts = append(parts, fmt.Sprintf("Extracted %d files (%s)", len(a.Files), formatBytes(a.ExtractedSize))) + } else { + parts = append(parts, fmt.Sprintf("Extracted %d files", len(a.Files))) + } + } + + if len(a.Directories) > 0 { + parts = append(parts, fmt.Sprintf("%d directories", len(a.Directories))) + } + + if len(a.Symlinks) > 0 { + parts = append(parts, fmt.Sprintf("%d symlinks", len(a.Symlinks))) + } + + // Source info + sourcePart := fmt.Sprintf("from %s", filepath.Base(a.Source)) + if a.CompressedSize > 0 { + sourcePart += fmt.Sprintf(" (%s)", formatBytes(a.CompressedSize)) + } + parts = append(parts, sourcePart) + + // Destination + parts = append(parts, fmt.Sprintf("to %s", a.Destination)) + + result := strings.Join(parts, ", ") + + // Add compression ratio if available + if a.CompressionRatio > 1 { + result += fmt.Sprintf("\nCompression ratio: %.2f:1", a.CompressionRatio) + } + + // Add issues if any + var issues []string + if len(a.Skipped) > 0 { + issues = append(issues, fmt.Sprintf("%d skipped", len(a.Skipped))) + } + if len(a.Errors) > 0 { + issues = append(issues, fmt.Sprintf("%d errors", len(a.Errors))) + } + if len(issues) > 0 { + result += fmt.Sprintf(" (%s)", strings.Join(issues, ", ")) + } + + return result +} + var blacklistedPathSymbols = "${}[]?*:<>|" var blockedPrefixes = []string{"/run/", "/proc/", "/etc/", "/var/", "/tmp/", "/dev/"} @@ -136,16 +235,35 @@ func GzipFile(path string) ([]byte, error) { return result, nil } -// Unarchive extracts the contents of an archive to the dest directory -func Unarchive(src, dest string) error { - logger.Debugf("Unarchiving %s to %s", src, dest) - if strings.HasSuffix(src, ".zip") { - return Unzip(src, dest) - } else if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tgz") || strings.HasSuffix(src, ".tar.gz") { - return Untar(src, dest) +// UnarchiveSimple extracts the contents of an archive to the dest directory (returns only error for backwards compatibility) +func UnarchiveSimple(src, dest string) error { + _, err := Unarchive(src, dest) + return err +} + +// Unarchive extracts an archive and returns detailed results with optional configuration +func Unarchive(src, dest string, options ...UnarchiveOption) (*Archive, error) { + // Apply default options + opts := &UnarchiveOptions{ + Overwrite: false, // Default: don't overwrite existing files + } + for _, option := range options { + option(opts) } - return fmt.Errorf("unknown format type %s", src) + logger.Debugf("Unarchiving %s to %s (overwrite=%v)", src, dest, opts.Overwrite) + if strings.HasSuffix(src, ".zip") || strings.HasSuffix(src, ".jar") { + return unzipWithResult(src, dest, opts) + } else if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tgz") || strings.HasSuffix(src, ".tar.gz") || strings.HasSuffix(src, ".tar.xz") || strings.HasSuffix(src, ".txz") { + return UntarWithFilterAndResult(src, dest, nil, opts) + } + + return nil, fmt.Errorf("unknown format type %s", src) +} + +// UnarchiveWithResult is deprecated, use Unarchive instead +func UnarchiveWithResult(src, dest string) (*Archive, error) { + return Unarchive(src, dest) } // UnarchiveExecutables extracts all executable's to the dest directory, ignoring any path's specified by the archive @@ -153,7 +271,7 @@ func UnarchiveExecutables(src, dest string) error { logger.Debugf("Unarchiving %s to %s", src, dest) if strings.HasSuffix(src, ".zip") { return Unzip(src, dest) - } else if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tgz") || strings.HasSuffix(src, ".tar.gz") || strings.HasSuffix(src, ".tar.xz") { + } else if strings.HasSuffix(src, ".tar") || strings.HasSuffix(src, ".tgz") || strings.HasSuffix(src, ".tar.gz") || strings.HasSuffix(src, ".tar.xz") || strings.HasSuffix(src, ".txz") { return UntarWithFilter(src, dest, func(header os.FileInfo) string { if fmt.Sprintf("%v", header.Mode()&0100) != "---x------" { return "" @@ -218,125 +336,398 @@ func Ungzip(source, target string) error { // FileFilter is a function used for filtering files type FileFilter func(header os.FileInfo) string -// Unzip the source file to the target directory -func Unzip(src, dest string) error { +// unzipWithResult extracts zip archive and returns detailed results +func unzipWithResult(src, dest string, opts *UnarchiveOptions) (*Archive, error) { + absSrc, err := filepath.Abs(src) + if err != nil { + return nil, fmt.Errorf("failed to resolve source path: %w", err) + } + absDest, err := filepath.Abs(dest) + if err != nil { + return nil, fmt.Errorf("failed to resolve destination path: %w", err) + } + + // Initialize archive result + archive := &Archive{ + Source: absSrc, + Destination: absDest, + Files: make([]string, 0), + Directories: make([]string, 0), + Symlinks: make([]string, 0), + Skipped: make([]string, 0), + Errors: make([]error, 0), + Overwritten: make([]string, 0), + } + + // Get compressed size from file stat + if stat, err := os.Stat(src); err == nil { + archive.CompressedSize = stat.Size() + } + + logger.V(3).Infof("Unzip: starting extraction of %s to %s", absSrc, absDest) + r, err := zip.OpenReader(src) if err != nil { - return err + return archive, err } defer r.Close() + if err := os.MkdirAll(dest, 0755); err != nil { + return archive, fmt.Errorf("failed to create target directory %s: %w", absDest, err) + } + + // Open the target directory as a root for secure file operations + root, err := os.OpenRoot(dest) + if err != nil { + return archive, fmt.Errorf("failed to open target directory as root %s: %w", absDest, err) + } + defer root.Close() + for _, f := range r.File { + if err := ValidatePath(f.Name); err != nil { + return archive, err + } + rc, err := f.Open() if err != nil { - return err + return archive, err } - defer rc.Close() - if err := ValidatePath(f.Name); err != nil { - return err - } + path := f.Name + info := f.FileInfo() - fpath := filepath.Join(dest, f.Name) - if f.FileInfo().IsDir() { - if err := os.MkdirAll(fpath, f.Mode()); err != nil { - return fmt.Errorf("failed to create directory %s: %w", fpath, err) + if info.IsDir() { + _ = rc.Close() + dirMode := info.Mode() & os.ModePerm + if dirMode == 0 { + dirMode = 0755 } - } else { - var fdir string - if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 { - fdir = fpath[:lastIndex] + if err := root.MkdirAll(path, dirMode); err != nil { + return archive, fmt.Errorf("failed to create directory %s: %w", path, err) } + archive.Directories = append(archive.Directories, path) + continue + } - err = os.MkdirAll(fdir, f.Mode()) + // Handle symlinks if present in zip + if info.Mode()&os.ModeSymlink != 0 { + linkData, err := io.ReadAll(rc) + _ = rc.Close() if err != nil { - return err + return archive, fmt.Errorf("failed to read symlink target for %s: %w", path, err) } - f, err := os.OpenFile( - fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err + linkTarget := string(linkData) + + // Validate symlink target - only allow relative paths within extraction dir + if filepath.IsAbs(linkTarget) || !filepath.IsLocal(linkTarget) { + return archive, fmt.Errorf("symlink target %s is absolute and not allowed", linkTarget) } - defer f.Close() - _, err = io.Copy(f, rc) - if err != nil { - return err + // Create parent directory + parent := filepath.Dir(path) + if parent != "." && parent != "" { + if err := root.MkdirAll(parent, 0755); err != nil { + return archive, fmt.Errorf("failed to create directory %s: %w", parent, err) + } + } + + if err := root.Symlink(linkTarget, path); err != nil { + return archive, fmt.Errorf("failed to create symlink %s -> %s: %w", path, linkTarget, err) + } + archive.Symlinks = append(archive.Symlinks, path) + continue + } + + // Regular file + // Create parent directory + parent := filepath.Dir(path) + if parent != "." && parent != "" { + if err := root.MkdirAll(parent, 0755); err != nil { + _ = rc.Close() + return archive, fmt.Errorf("failed to create directory %s: %w", parent, err) + } + } + + // Check for existing file + fileExists := false + if _, err := root.Stat(path); err == nil { + fileExists = true + if !opts.Overwrite { + _ = rc.Close() + return archive, fmt.Errorf("file %s already exists", path) } + archive.Overwritten = append(archive.Overwritten, path) } + + // Open file with appropriate flags + flags := os.O_CREATE | os.O_RDWR + if fileExists && opts.Overwrite { + flags |= os.O_TRUNC + } + file, err := root.OpenFile(path, flags, info.Mode()) + if err != nil { + _ = rc.Close() + return archive, fmt.Errorf("failed to create file %s: %w", path, err) + } + + bytesWritten, err := io.Copy(file, rc) + _ = file.Close() + _ = rc.Close() + if err != nil { + return archive, fmt.Errorf("failed to write file %s: %w", path, err) + } + + archive.Files = append(archive.Files, path) + archive.ExtractedSize += bytesWritten + } + + // Calculate compression ratio + if archive.CompressedSize > 0 && archive.ExtractedSize > 0 { + archive.CompressionRatio = float64(archive.ExtractedSize) / float64(archive.CompressedSize) } - return nil + logger.V(3).Infof("Unzip: extraction complete for %s", archive) + return archive, nil } -// Untar extracts all files in tarball to the target directory +// Unzip the source file to the target directory (backwards compatibility wrapper) +func Unzip(src, dest string) error { + _, err := Unarchive(src, dest) + return err +} + + +// Untar extracts all files in tarball to the target directory (backwards compatibility wrapper) func Untar(tarball, target string) error { - return UntarWithFilter(tarball, target, nil) + _, err := Unarchive(tarball, target) + return err } // UntarWithFilter extracts all files in tarball to the target directory, passing each file to filter // if the filter returns "" then the file is ignored, otherwise the return string is used as the relative // destination path func UntarWithFilter(tarball, target string, filter FileFilter) error { + opts := &UnarchiveOptions{Overwrite: true} // Maintain backward compatibility + _, err := UntarWithFilterAndResult(tarball, target, filter, opts) + return err +} + +// UntarWithResult extracts all files in tarball to the target directory and returns detailed results +func UntarWithResult(tarball, target string) (*Archive, error) { + opts := &UnarchiveOptions{Overwrite: true} // Maintain backward compatibility + return UntarWithFilterAndResult(tarball, target, nil, opts) +} + +// UntarWithFilterAndResult extracts all files in tarball with filter and returns detailed results +func UntarWithFilterAndResult(tarball, target string, filter FileFilter, opts *UnarchiveOptions) (*Archive, error) { + absTarball, err := filepath.Abs(tarball) + if err != nil { + return nil, fmt.Errorf("failed to resolve tarball path: %w", err) + } + absTarget, err := filepath.Abs(target) + if err != nil { + return nil, fmt.Errorf("failed to resolve target path: %w", err) + } + + // Initialize archive result + archive := &Archive{ + Source: absTarball, + Destination: absTarget, + Files: make([]string, 0), + Directories: make([]string, 0), + Symlinks: make([]string, 0), + Skipped: make([]string, 0), + Errors: make([]error, 0), + Overwritten: make([]string, 0), + } + + // Get compressed size from file stat + if stat, err := os.Stat(tarball); err == nil { + archive.CompressedSize = stat.Size() + } + + logger.V(3).Infof("Untar: starting extraction of %s to %s", absTarball, absTarget) + var reader io.Reader file, err := os.Open(tarball) if err != nil { - return err + return archive, err } defer file.Close() reader = file + + // Detect and handle compression if strings.HasSuffix(tarball, ".tar.gz") || strings.HasSuffix(tarball, ".tgz") { reader, err = gzip.NewReader(reader) if err != nil { - return err + return archive, fmt.Errorf("failed to create gzip reader: %w", err) } - } else if strings.HasSuffix(tarball, ".tar.xz") { + } else if strings.HasSuffix(tarball, ".tar.xz") || strings.HasSuffix(tarball, ".txz") { reader, err = xz.NewReader(reader) if err != nil { - return err + return archive, fmt.Errorf("failed to create xz reader: %w", err) } } tarReader := tar.NewReader(reader) + if err := os.MkdirAll(target, 0755); err != nil { + return archive, fmt.Errorf("failed to create target directory %s: %w", absTarget, err) + } + + // Open the target directory as a root for secure file operations + root, err := os.OpenRoot(target) + if err != nil { + return archive, fmt.Errorf("failed to open target directory as root %s: %w", absTarget, err) + } + defer root.Close() + for { header, err := tarReader.Next() if err == io.EOF { break } else if err != nil { - return err + return archive, fmt.Errorf("error reading tar entry: %w", err) } info := header.FileInfo() + path := header.Name if err := ValidatePath(header.Name); err != nil { - return err + return archive, err } - path := filepath.Join(target, header.Name) + if filter != nil { - path = filter(info) - if path == "" { + fp := filter(info) + if fp == "" { + archive.Skipped = append(archive.Skipped, path) continue } - path = filepath.Join(target, path) } + if info.IsDir() { - if err = os.MkdirAll(path, info.Mode()); err != nil { - return err + dirMode := info.Mode() & os.ModePerm // Extract permission bits only + if dirMode == 0 { + dirMode = 0755 // Default directory permissions + } + if err = root.MkdirAll(path, dirMode); err != nil { + return archive, fmt.Errorf("failed to create directory %s: %w", path, err) } + archive.Directories = append(archive.Directories, path) continue } - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) - if err != nil { - return err + parent := filepath.Dir(path) + if parent != "." && parent != "" { + if err := root.MkdirAll(parent, 0755); err != nil { + return archive, fmt.Errorf("failed to create directory %s: %w", parent, err) + } } - defer file.Close() - _, err = io.Copy(file, tarReader) - if err != nil { - return err + + switch header.Typeflag { + case tar.TypeReg: + // Check for existing file + fileExists := false + if _, err := root.Stat(path); err == nil { + fileExists = true + if !opts.Overwrite { + return archive, fmt.Errorf("file %s already exists", path) + } + archive.Overwritten = append(archive.Overwritten, path) + } + + // Open file with appropriate flags + flags := os.O_CREATE | os.O_RDWR + if fileExists && opts.Overwrite { + flags |= os.O_TRUNC + } + file, err := root.OpenFile(path, flags, os.FileMode(header.Mode)) + if err != nil { + return archive, fmt.Errorf("failed to create file %s (mode=%v) %s", path, info.Mode(), err) + } + bytesWritten, err := io.Copy(file, tarReader) + _ = file.Close() + if err != nil { + return archive, fmt.Errorf("failed to write file %s: %w", path, err) + } + archive.Files = append(archive.Files, path) + archive.ExtractedSize += bytesWritten + + case tar.TypeSymlink: + // Validate the symlink target stays within extraction dir + linkTarget := header.Linkname + // Only check relative paths; absolute always forbidden + if filepath.IsAbs(linkTarget) || !filepath.IsLocal(linkTarget) { + return archive, fmt.Errorf("symlink target %s is absolute and not allowed", linkTarget) + } + + if err := root.Symlink(linkTarget, path); err != nil { + return archive, fmt.Errorf("failed to create symlink %s -> %s: %w", path, linkTarget, err) + } + archive.Symlinks = append(archive.Symlinks, path) + + case tar.TypeDir: + dirMode := os.FileMode(header.Mode) & os.ModePerm // Extract permission bits only + if dirMode == 0 { + dirMode = 0755 // Default directory permissions + } + if err := root.MkdirAll(path, dirMode); err != nil { + return archive, fmt.Errorf("failed to create directory %s: %w", path, err) + } + archive.Directories = append(archive.Directories, path) + + case tar.TypeLink: + // Hard link - copy the target file content + linkTarget := header.Linkname + targetFile, err := root.Open(linkTarget) + if err != nil { + return nil, fmt.Errorf("cannot read hard link target %s for %s: %v", linkTarget, path, err) + } + + // Check for existing file + fileExists := false + if _, err := root.Stat(path); err == nil { + fileExists = true + if !opts.Overwrite { + _ = targetFile.Close() + return archive, fmt.Errorf("file %s already exists", path) + } + archive.Overwritten = append(archive.Overwritten, path) + } + + // Open file with appropriate flags + flags := os.O_CREATE | os.O_RDWR + if fileExists && opts.Overwrite { + flags |= os.O_TRUNC + } + file, err := root.OpenFile(path, flags, os.FileMode(header.Mode)) + if err != nil { + _ = targetFile.Close() + return nil, fmt.Errorf("failed to create hard link file %s (mode=%v): %w", path, info.Mode(), err) + } + + bytesWritten, err := io.Copy(file, targetFile) + _ = targetFile.Close() + _ = file.Close() + + if err != nil { + return archive, fmt.Errorf("failed to copy content for hard link %s: %w", path, err) + } + archive.Files = append(archive.Files, path) + archive.ExtractedSize += bytesWritten + + default: + return nil, fmt.Errorf("unsupported file type %c (%d) for %s", header.Typeflag, header.Typeflag, path) } } - return nil + + // Calculate compression ratio + if archive.CompressedSize > 0 && archive.ExtractedSize > 0 { + archive.CompressionRatio = float64(archive.ExtractedSize) / float64(archive.CompressedSize) + } + + logger.V(3).Infof("Untar: extraction complete for %s", archive) + + return archive, nil } // SafeRead reads a path and returns the text contents or nil diff --git a/files/testdata/archives/corrupted.tar.gz b/files/testdata/archives/corrupted.tar.gz new file mode 100644 index 0000000..d02d330 Binary files /dev/null and b/files/testdata/archives/corrupted.tar.gz differ diff --git a/files/testdata/archives/dirs_only.tar b/files/testdata/archives/dirs_only.tar new file mode 100644 index 0000000..cba61bb Binary files /dev/null and b/files/testdata/archives/dirs_only.tar differ diff --git a/files/testdata/archives/empty.tar b/files/testdata/archives/empty.tar new file mode 100644 index 0000000..9df6499 Binary files /dev/null and b/files/testdata/archives/empty.tar differ diff --git a/files/testdata/archives/executable.tar b/files/testdata/archives/executable.tar new file mode 100644 index 0000000..1c6421b Binary files /dev/null and b/files/testdata/archives/executable.tar differ diff --git a/files/testdata/archives/hardlinks.tar b/files/testdata/archives/hardlinks.tar new file mode 100644 index 0000000..7229d33 Binary files /dev/null and b/files/testdata/archives/hardlinks.tar differ diff --git a/files/testdata/archives/nested.tar b/files/testdata/archives/nested.tar new file mode 100644 index 0000000..3e5449c Binary files /dev/null and b/files/testdata/archives/nested.tar differ diff --git a/files/testdata/archives/nested.tgz b/files/testdata/archives/nested.tgz new file mode 100644 index 0000000..e91e3cb Binary files /dev/null and b/files/testdata/archives/nested.tgz differ diff --git a/files/testdata/archives/nested.txz b/files/testdata/archives/nested.txz new file mode 100644 index 0000000..c32928b Binary files /dev/null and b/files/testdata/archives/nested.txz differ diff --git a/files/testdata/archives/parent_symlinks.tar b/files/testdata/archives/parent_symlinks.tar new file mode 100644 index 0000000..8623204 Binary files /dev/null and b/files/testdata/archives/parent_symlinks.tar differ diff --git a/files/testdata/archives/safe_postgres.tar.xz b/files/testdata/archives/safe_postgres.tar.xz new file mode 100644 index 0000000..542a2ab Binary files /dev/null and b/files/testdata/archives/safe_postgres.tar.xz differ diff --git a/files/testdata/archives/safe_symlinks.tar b/files/testdata/archives/safe_symlinks.tar new file mode 100644 index 0000000..0847c87 Binary files /dev/null and b/files/testdata/archives/safe_symlinks.tar differ diff --git a/files/testdata/archives/simple.tar b/files/testdata/archives/simple.tar new file mode 100644 index 0000000..cda8dda Binary files /dev/null and b/files/testdata/archives/simple.tar differ diff --git a/files/testdata/archives/simple.tar.gz b/files/testdata/archives/simple.tar.gz new file mode 100644 index 0000000..45bb712 Binary files /dev/null and b/files/testdata/archives/simple.tar.gz differ diff --git a/files/testdata/archives/simple.tar.xz b/files/testdata/archives/simple.tar.xz new file mode 100644 index 0000000..b6c6983 Binary files /dev/null and b/files/testdata/archives/simple.tar.xz differ diff --git a/files/testdata/archives/simple.zip b/files/testdata/archives/simple.zip new file mode 100644 index 0000000..376f9c1 Binary files /dev/null and b/files/testdata/archives/simple.zip differ diff --git a/files/testdata/archives/special_chars.zip b/files/testdata/archives/special_chars.zip new file mode 100644 index 0000000..cf5d6b9 Binary files /dev/null and b/files/testdata/archives/special_chars.zip differ diff --git a/files/testdata/archives/symlinks.tar.xz b/files/testdata/archives/symlinks.tar.xz new file mode 100644 index 0000000..97f7d2a Binary files /dev/null and b/files/testdata/archives/symlinks.tar.xz differ diff --git a/files/testdata/archives/test.jar b/files/testdata/archives/test.jar new file mode 100644 index 0000000..376f9c1 Binary files /dev/null and b/files/testdata/archives/test.jar differ diff --git a/files/testdata/executables/regular.txt b/files/testdata/executables/regular.txt new file mode 100644 index 0000000..00cb5bc --- /dev/null +++ b/files/testdata/executables/regular.txt @@ -0,0 +1 @@ +regular content diff --git a/files/testdata/executables/script.sh b/files/testdata/executables/script.sh new file mode 100755 index 0000000..6481942 --- /dev/null +++ b/files/testdata/executables/script.sh @@ -0,0 +1,2 @@ +#!/bin/bash +echo hello diff --git a/files/testdata/hardlinks_test/hardlink.txt b/files/testdata/hardlinks_test/hardlink.txt new file mode 100644 index 0000000..2bdd920 --- /dev/null +++ b/files/testdata/hardlinks_test/hardlink.txt @@ -0,0 +1 @@ +Original file content diff --git a/files/testdata/hardlinks_test/original.txt b/files/testdata/hardlinks_test/original.txt new file mode 100644 index 0000000..2bdd920 --- /dev/null +++ b/files/testdata/hardlinks_test/original.txt @@ -0,0 +1 @@ +Original file content diff --git a/files/testdata/postgres_safe/bin/config_link b/files/testdata/postgres_safe/bin/config_link new file mode 120000 index 0000000..afb4a21 --- /dev/null +++ b/files/testdata/postgres_safe/bin/config_link @@ -0,0 +1 @@ +postgres.conf \ No newline at end of file diff --git a/files/testdata/postgres_safe/bin/postgres b/files/testdata/postgres_safe/bin/postgres new file mode 100644 index 0000000..78bcafc --- /dev/null +++ b/files/testdata/postgres_safe/bin/postgres @@ -0,0 +1 @@ +postgres binary diff --git a/files/testdata/postgres_safe/bin/postgres.conf b/files/testdata/postgres_safe/bin/postgres.conf new file mode 100644 index 0000000..04204c7 --- /dev/null +++ b/files/testdata/postgres_safe/bin/postgres.conf @@ -0,0 +1 @@ +config diff --git a/files/testdata/safe_symlinks/safe_link b/files/testdata/safe_symlinks/safe_link new file mode 120000 index 0000000..4cbb553 --- /dev/null +++ b/files/testdata/safe_symlinks/safe_link @@ -0,0 +1 @@ +target.txt \ No newline at end of file diff --git a/files/testdata/safe_symlinks/target.txt b/files/testdata/safe_symlinks/target.txt new file mode 100644 index 0000000..2ceb84c --- /dev/null +++ b/files/testdata/safe_symlinks/target.txt @@ -0,0 +1 @@ +target content diff --git a/files/testdata/source/file1.txt b/files/testdata/source/file1.txt new file mode 100644 index 0000000..679d87b --- /dev/null +++ b/files/testdata/source/file1.txt @@ -0,0 +1,2 @@ +This is file1 content. +It has multiple lines. \ No newline at end of file diff --git a/files/testdata/source/file2.txt b/files/testdata/source/file2.txt new file mode 100644 index 0000000..60e4240 --- /dev/null +++ b/files/testdata/source/file2.txt @@ -0,0 +1 @@ +File2 has different content. \ No newline at end of file diff --git a/files/testdata/source/parent_symlinks/parent_traversal b/files/testdata/source/parent_symlinks/parent_traversal new file mode 120000 index 0000000..53231de --- /dev/null +++ b/files/testdata/source/parent_symlinks/parent_traversal @@ -0,0 +1 @@ +../source/file1.txt \ No newline at end of file diff --git a/files/testdata/source/parent_symlinks/test.txt b/files/testdata/source/parent_symlinks/test.txt new file mode 100644 index 0000000..d670460 --- /dev/null +++ b/files/testdata/source/parent_symlinks/test.txt @@ -0,0 +1 @@ +test content diff --git a/files/testdata/source/subdir/file3.txt b/files/testdata/source/subdir/file3.txt new file mode 100644 index 0000000..a076e91 --- /dev/null +++ b/files/testdata/source/subdir/file3.txt @@ -0,0 +1 @@ +This is file3 in a subdirectory. \ No newline at end of file diff --git a/files/testdata/special_chars/file with spaces.txt b/files/testdata/special_chars/file with spaces.txt new file mode 100644 index 0000000..ac3e272 --- /dev/null +++ b/files/testdata/special_chars/file with spaces.txt @@ -0,0 +1 @@ +content1 diff --git a/files/testdata/special_chars/file_with_underscore.txt b/files/testdata/special_chars/file_with_underscore.txt new file mode 100644 index 0000000..637f034 --- /dev/null +++ b/files/testdata/special_chars/file_with_underscore.txt @@ -0,0 +1 @@ +content2 diff --git a/go.mod b/go.mod index 3360cf7..24ce35a 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/flanksource/commons -go 1.23.0 - -toolchain go1.23.4 +go 1.25.1 require ( + github.com/Masterminds/semver/v3 v3.4.0 github.com/Snawoot/go-http-digest-auth-client v1.1.3 github.com/bsm/gomega v1.27.10 github.com/emirpasic/gods/v2 v2.0.0-alpha @@ -31,7 +30,6 @@ require ( github.com/shirou/gopsutil/v3 v3.24.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.6 - github.com/stretchr/testify v1.10.0 github.com/ulikunitz/xz v0.5.12 github.com/vadimi/go-http-ntlm v1.0.3 github.com/vadimi/go-http-ntlm/v2 v2.5.0 diff --git a/go.sum b/go.sum index decbe53..ab86ce2 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Snawoot/go-http-digest-auth-client v1.1.3 h1:Xd/SNBuIUJqotzmxRpbXovBJxmlVZOT19IZZdMdrJ0Q= github.com/Snawoot/go-http-digest-auth-client v1.1.3/go.mod h1:WiwNiPXTRGyjTGpBtSQJlM2wDPRRPpFGhMkMWpV4uqg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/http/request.go b/http/request.go index fa3ed42..c8c2a8c 100644 --- a/http/request.go +++ b/http/request.go @@ -245,7 +245,7 @@ func (r *Request) Debug() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s %s\n", r.method, logger.StripSecrets(r.url.String()))) for k, v := range logger.StripSecretsFromMap(r.HeaderMap()) { - sb.WriteString(fmt.Sprintf(" %s: %s\n", console.Grayf(k), v)) + sb.WriteString(fmt.Sprintf(" %s: %s\n", console.Grayf("%s", k), v)) } body, _ := io.ReadAll(r.body) sb.WriteString(logger.StripSecrets(string(body))) diff --git a/http/response.go b/http/response.go index 1aa43ee..3ef5a14 100644 --- a/http/response.go +++ b/http/response.go @@ -111,7 +111,7 @@ func (h *Response) Debug() string { sb.WriteString(fmt.Sprintf("\n====> Status: %d\n", h.StatusCode)) for k, v := range logger.StripSecretsFromMap(h.HeaderMap()) { - sb.WriteString(fmt.Sprintf(" %s: %s\n", console.Grayf(k), v)) + sb.WriteString(fmt.Sprintf(" %s: %s\n", console.Grayf("%s", k), v)) } if h.IsJSON() { r, _ := h.AsJSON() diff --git a/is/is.go b/is/is.go index 7ff6946..0b833c3 100644 --- a/is/is.go +++ b/is/is.go @@ -28,5 +28,6 @@ func Archive(filename string) bool { return strings.HasSuffix(filename, ".zip") || strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".gz") || - strings.HasSuffix(filename, ".xz") + strings.HasSuffix(filename, ".xz") || + strings.HasSuffix(filename, ".txz") } diff --git a/logger/buffered.go b/logger/buffered.go new file mode 100644 index 0000000..52d0ad7 --- /dev/null +++ b/logger/buffered.go @@ -0,0 +1,432 @@ +package logger + +import ( + "fmt" + "log/slog" + "strings" + "sync" + "time" +) + +// BufferedLogEntry represents a single buffered log message +type BufferedLogEntry struct { + Message string + Time time.Time + Level LogLevel +} + +// BufferedLogger implements Logger interface with in-memory log storage +type BufferedLogger struct { + mu sync.RWMutex + logsByLevel map[LogLevel][]BufferedLogEntry + maxLogsByLevel map[LogLevel]int + logLevel LogLevel +} + +// NewBufferedLogger creates a new BufferedLogger with default retention strategy +func NewBufferedLogger(maxLogs int) *BufferedLogger { + return NewBufferedLoggerWithRetention(getDefaultRetentionConfig(maxLogs)) +} + +// RetentionConfig defines retention limits for each log level +type RetentionConfig map[LogLevel]int + +// NewBufferedLoggerWithRetention creates a new BufferedLogger with custom retention config +func NewBufferedLoggerWithRetention(config RetentionConfig) *BufferedLogger { + logsByLevel := make(map[LogLevel][]BufferedLogEntry) + maxLogsByLevel := make(map[LogLevel]int) + + // Initialize buffers for each configured level + for level, maxCount := range config { + logsByLevel[level] = make([]BufferedLogEntry, 0, maxCount) + maxLogsByLevel[level] = maxCount + } + + return &BufferedLogger{ + logsByLevel: logsByLevel, + maxLogsByLevel: maxLogsByLevel, + logLevel: Info, + } +} + +// getDefaultRetentionConfig returns default retention limits based on total maxLogs +func getDefaultRetentionConfig(maxLogs int) RetentionConfig { + if maxLogs <= 0 { + maxLogs = 50 // Default total + } + + // Base retention strategy + baseConfig := map[LogLevel]int{ + Fatal: maxLogs * 20 / 100, // 20% for Fatal + Error: maxLogs * 25 / 100, // 25% for Error + Warn: maxLogs * 20 / 100, // 20% for Warn + Info: maxLogs * 20 / 100, // 20% for Info + Debug: maxLogs * 10 / 100, // 10% for Debug + Trace: maxLogs * 5 / 100, // 5% for Trace + } + + // Ensure minimum of 1 for each level + for level := range baseConfig { + if baseConfig[level] < 1 { + baseConfig[level] = 1 + } + } + + return baseConfig +} + +// appendLog adds a log entry to the appropriate level buffer +func (b *BufferedLogger) appendLog(level LogLevel, format string, args ...interface{}) { + b.mu.Lock() + defer b.mu.Unlock() + + entry := BufferedLogEntry{ + Message: fmt.Sprintf(format, args...), + Time: time.Now(), + Level: level, + } + + // Get or create buffer for this level + if _, exists := b.logsByLevel[level]; !exists { + // Initialize with default retention if not configured + maxCount := 10 // Default + if configured, ok := b.maxLogsByLevel[level]; ok { + maxCount = configured + } + b.logsByLevel[level] = make([]BufferedLogEntry, 0, maxCount) + b.maxLogsByLevel[level] = maxCount + } + + // Add to appropriate level buffer + b.logsByLevel[level] = append(b.logsByLevel[level], entry) + + // Trim if we exceed max logs for this level + maxForLevel := b.maxLogsByLevel[level] + if len(b.logsByLevel[level]) > maxForLevel { + b.logsByLevel[level] = b.logsByLevel[level][len(b.logsByLevel[level])-maxForLevel:] + } +} + +// GetLogs returns a copy of all buffered log entries, sorted by timestamp +func (b *BufferedLogger) GetLogs() []BufferedLogEntry { + b.mu.RLock() + defer b.mu.RUnlock() + + // Collect all entries from all level buffers + var allEntries []BufferedLogEntry + for _, levelEntries := range b.logsByLevel { + allEntries = append(allEntries, levelEntries...) + } + + // Sort by timestamp (oldest first) + for i := 0; i < len(allEntries)-1; i++ { + for j := i + 1; j < len(allEntries); j++ { + if allEntries[i].Time.After(allEntries[j].Time) { + allEntries[i], allEntries[j] = allEntries[j], allEntries[i] + } + } + } + + return allEntries +} + +// ClearLogs clears all buffered log entries +func (b *BufferedLogger) ClearLogs() { + b.mu.Lock() + defer b.mu.Unlock() + for level := range b.logsByLevel { + b.logsByLevel[level] = b.logsByLevel[level][:0] + } +} + +// GetLogsByLevel returns buffered log entries for a specific level +func (b *BufferedLogger) GetLogsByLevel(level LogLevel) []BufferedLogEntry { + b.mu.RLock() + defer b.mu.RUnlock() + + if entries, exists := b.logsByLevel[level]; exists { + result := make([]BufferedLogEntry, len(entries)) + copy(result, entries) + return result + } + return []BufferedLogEntry{} +} + +// SetRetentionPolicy updates the retention limits for log levels +func (b *BufferedLogger) SetRetentionPolicy(config RetentionConfig) { + b.mu.Lock() + defer b.mu.Unlock() + + for level, maxCount := range config { + b.maxLogsByLevel[level] = maxCount + + // If buffer already exists and exceeds new limit, trim it + if entries, exists := b.logsByLevel[level]; exists && len(entries) > maxCount { + b.logsByLevel[level] = entries[len(entries)-maxCount:] + } + } +} + +// GetRetentionPolicy returns current retention limits +func (b *BufferedLogger) GetRetentionPolicy() RetentionConfig { + b.mu.RLock() + defer b.mu.RUnlock() + + config := make(RetentionConfig) + for level, maxCount := range b.maxLogsByLevel { + config[level] = maxCount + } + return config +} + +// ScaleRetentionByLogLevel adjusts retention based on current log level +// Higher verbosity = more retention for all levels +func (b *BufferedLogger) ScaleRetentionByLogLevel() { + b.mu.Lock() + defer b.mu.Unlock() + + // Calculate scaling factor based on current log level + var scaleFactor float64 = 1.0 + switch { + case b.logLevel >= Trace4: + scaleFactor = 3.0 // Very verbose, keep lots of logs + case b.logLevel >= Trace2: + scaleFactor = 2.5 + case b.logLevel >= Trace: + scaleFactor = 2.0 + case b.logLevel >= Debug: + scaleFactor = 1.5 + default: + scaleFactor = 1.0 // Base retention for Info and below + } + + // Apply scaling to all configured levels + baseConfig := getDefaultRetentionConfig(50) // Use base config + for level, baseCount := range baseConfig { + newCount := int(float64(baseCount) * scaleFactor) + if newCount < 1 { + newCount = 1 + } + b.maxLogsByLevel[level] = newCount + + // Trim existing buffers if they exceed new limits + if entries, exists := b.logsByLevel[level]; exists && len(entries) > newCount { + b.logsByLevel[level] = entries[len(entries)-newCount:] + } + } +} + +// Warnf logs a warning message +func (b *BufferedLogger) Warnf(format string, args ...interface{}) { + b.appendLog(Warn, format, args...) +} + +// Infof logs an info message +func (b *BufferedLogger) Infof(format string, args ...interface{}) { + b.appendLog(Info, format, args...) +} + +// Errorf logs an error message +func (b *BufferedLogger) Errorf(format string, args ...interface{}) { + b.appendLog(Error, format, args...) +} + +// Debugf logs a debug message +func (b *BufferedLogger) Debugf(format string, args ...interface{}) { + if b.IsDebugEnabled() { + b.appendLog(Debug, format, args...) + } +} + +// Tracef logs a trace message +func (b *BufferedLogger) Tracef(format string, args ...interface{}) { + if b.IsTraceEnabled() { + b.appendLog(Trace, format, args...) + } +} + +// Fatalf logs a fatal message and panics +func (b *BufferedLogger) Fatalf(format string, args ...interface{}) { + b.appendLog(Fatal, format, args...) + panic(fmt.Sprintf(format, args...)) +} + +// WithValues returns the same logger (noop as requested) +func (b *BufferedLogger) WithValues(keysAndValues ...interface{}) Logger { + return b +} + +// IsTraceEnabled checks if trace level is enabled +func (b *BufferedLogger) IsTraceEnabled() bool { + b.mu.RLock() + defer b.mu.RUnlock() + return b.logLevel >= Trace +} + +// IsDebugEnabled checks if debug level is enabled +func (b *BufferedLogger) IsDebugEnabled() bool { + b.mu.RLock() + defer b.mu.RUnlock() + return b.logLevel >= Debug +} + +// IsLevelEnabled checks if a specific level is enabled +func (b *BufferedLogger) IsLevelEnabled(level LogLevel) bool { + b.mu.RLock() + defer b.mu.RUnlock() + return b.logLevel >= level +} + +// GetLevel returns the current log level +func (b *BufferedLogger) GetLevel() LogLevel { + b.mu.RLock() + defer b.mu.RUnlock() + return b.logLevel +} + +// SetLogLevel sets the log level and automatically scales retention +func (b *BufferedLogger) SetLogLevel(level any) { + b.mu.Lock() + defer b.mu.Unlock() + + oldLevel := b.logLevel + + switch v := level.(type) { + case LogLevel: + b.logLevel = v + case int: + b.logLevel = LogLevel(v) + case string: + // Parse string level + switch v { + case "trace": + b.logLevel = Trace + case "debug": + b.logLevel = Debug + case "info": + b.logLevel = Info + case "warn": + b.logLevel = Warn + case "error": + b.logLevel = Error + case "fatal": + b.logLevel = Fatal + default: + b.logLevel = Info + } + default: + b.logLevel = Info + } + + // Auto-scale retention if log level changed + if oldLevel != b.logLevel { + b.mu.Unlock() // Unlock temporarily for ScaleRetentionByLogLevel + b.ScaleRetentionByLogLevel() + b.mu.Lock() // Re-lock for defer + } +} + +// SetMinLogLevel sets the minimum log level (same as SetLogLevel for BufferedLogger) +func (b *BufferedLogger) SetMinLogLevel(level any) { + b.SetLogLevel(level) +} + +// V returns a verbose logger +func (b *BufferedLogger) V(level any) Verbose { + return &bufferedVerbose{ + logger: b, + enabled: b.IsLevelEnabled(ParseLevel(b, level)), + filters: nil, + } +} + +// WithV returns the same logger (for simplicity) +func (b *BufferedLogger) WithV(level any) Logger { + return b +} + +// Named returns the same logger (noop as requested) +func (b *BufferedLogger) Named(name string) Logger { + return b +} + +// WithoutName returns the same logger (noop as requested) +func (b *BufferedLogger) WithoutName() Logger { + return b +} + +// WithSkipReportLevel returns the same logger (noop as requested) +func (b *BufferedLogger) WithSkipReportLevel(i int) Logger { + return b +} + +// GetSlogLogger returns nil (unsupported as requested) +func (b *BufferedLogger) GetSlogLogger() *slog.Logger { + return nil +} + +// bufferedVerbose implements the Verbose interface for BufferedLogger +type bufferedVerbose struct { + logger *BufferedLogger + enabled bool + filters []string +} + +// isFiltered checks if a log line should be filtered out based on filters +func (v *bufferedVerbose) isFiltered(line string) bool { + if len(strings.TrimSpace(line)) == 0 { + return true + } + for _, filter := range v.filters { + if strings.Contains(line, filter) { + return true + } + } + return false +} + +// Write implements io.Writer interface +func (v *bufferedVerbose) Write(p []byte) (n int, err error) { + if !v.enabled { + return len(p), nil + } + + for _, line := range strings.Split(string(p), "\n") { + if v.isFiltered(line) { + continue + } + v.logger.Infof("%s", line) + } + + return len(p), nil +} + +// Infof logs an info message if enabled and not filtered +func (v *bufferedVerbose) Infof(format string, args ...interface{}) { + if !v.enabled { + return + } + + message := fmt.Sprintf(format, args...) + if !v.isFiltered(message) { + v.logger.Infof("%s", message) + } +} + +// WithFilter returns a new verbose logger with additional filters +func (v *bufferedVerbose) WithFilter(filters ...string) Verbose { + newFilters := make([]string, len(v.filters)+len(filters)) + copy(newFilters, v.filters) + copy(newFilters[len(v.filters):], filters) + + return &bufferedVerbose{ + logger: v.logger, + enabled: v.enabled, + filters: newFilters, + } +} + +// Enabled returns whether this verbose logger is enabled +func (v *bufferedVerbose) Enabled() bool { + return v.enabled +} diff --git a/logger/default.go b/logger/default.go index 962a7a0..34ecdfe 100644 --- a/logger/default.go +++ b/logger/default.go @@ -32,6 +32,8 @@ func (f *flagSet) bindFlags(flags *pflag.FlagSet) { func (f *flagSet) Parse() error { logFlagset := pflag.NewFlagSet("logger", pflag.ContinueOnError) + logFlagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true} + // standalone parsing of flags to ensure we always have the correct values f.bindFlags(logFlagset) if err := logFlagset.Parse(os.Args[1:]); err != nil { @@ -41,6 +43,7 @@ func (f *flagSet) Parse() error { re, _ := regexp.Compile("-v{1,}") for _, arg := range os.Args[1:] { // FIXME there seems to be a race condition where pflag + // will return a count that does not match the actual number of -v flags if strings.HasPrefix(arg, "-v") { if strings.Contains(arg, "=") { diff --git a/logger/slog.go b/logger/slog.go index 6debe8b..b908bd9 100644 --- a/logger/slog.go +++ b/logger/slog.go @@ -228,7 +228,7 @@ func (s SlogLogger) Infof(format string, args ...interface{}) { } func (s SlogLogger) Secretf(format string, args ...interface{}) { - s.Debugf(StripSecrets(fmt.Sprintf(format, args...))) + s.Debugf("%s", StripSecrets(fmt.Sprintf(format, args...))) } func (s SlogLogger) Prettyf(msg string, obj interface{}) { @@ -265,7 +265,7 @@ func (s SlogLogger) handleRaw(r slog.Record, msg string) { } r.Message = msg } else if s.Prefix != "" { - r.Message = fmt.Sprintf("(%s) %s", BrightF(s.Prefix), msg) + r.Message = fmt.Sprintf("(%s) %s", BrightF("%s", s.Prefix), msg) } else { r.Message = msg }