diff --git a/.gitignore b/.gitignore index 7069012..fa8aaaf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ *.out # Intellij -.idea/ \ No newline at end of file +.idea/ +.vscode/ +cliff.toml diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d705ea0 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,87 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +flanksource/commons is a Go utility library providing common functionality for the Flanksource ecosystem. It's a collection of reusable modules organized by concern: core utilities, infrastructure helpers, development tools, data processing, and system integration. + +## Key Commands + +```bash +# Testing +make test # Run all tests with verbose output +go test ./... # Run all tests (standard Go) +go test ./logger/... # Run tests for a specific package + +# Linting and Code Quality +make lint # Run golangci-lint with project-specific rules + +# Dependency Management +make tidy # Clean up go.mod dependencies +go mod tidy # Standard Go dependency cleanup + +# Build +go build # Build the dependency installer binary +go run main.go # Download and install a dependency (e.g., go run main.go jq) +``` + +## Architecture and Code Organization + +### Module Structure +The codebase follows a flat package structure where each directory represents a focused utility module: + +- **HTTP Client (`http/`)**: Enhanced HTTP client with authentication middleware pattern. Supports Digest, NTLM, OAuth2, and custom auth. Uses functional options for configuration. +- **Logger (`logger/`)**: Dual logging system supporting both logrus and slog backends. Global logger instance with interface-based design for flexibility. +- **Collections (`collections/`)**: Generic utilities for maps, slices, sets, and priority queues. Heavy use of Go generics. +- **Dependency Management (`deps/`)**: Template-based system for downloading and installing external binaries. Supports cross-platform paths and verification. +- **Properties (`properties/`)**: Global configuration management with environment variable support and structured properties. + +### Key Patterns + +1. **Global State Pattern**: Logger and Properties packages use global instances initialized via `Init()` functions. Always check if initialization is needed before using these packages. + +2. **Middleware Pattern**: HTTP client uses layered middleware for auth, tracing, and logging: + ```go + client := http.NewClient(). + WithLogger(logger). + WithAuth(username, password). + WithTrace(true) + ``` + +3. **Interface-First Design**: Most packages expose interfaces (e.g., `Logger` interface) allowing for mock implementations in tests. + +4. **Builder Pattern**: Used extensively in HTTP client and dependency configurations for fluent API design. + +### Testing Approach + +- Uses testify for assertions in most tests +- Some modules (like `diff/`) use Ginkgo/Gomega for BDD-style tests +- Test files are colocated with source files following `*_test.go` convention +- No global test setup/teardown - each test is self-contained + +### Important Development Notes + +1. **Go Version**: Requires Go 1.23.0+ (uses newer generic features) +2. **Linting**: The project has custom golangci-lint rules that disable certain checks for HTTP-related naming +3. **Imports**: When adding new functionality, prefer using existing utilities from within the project (e.g., use `collections` package for slice operations) +4. **Error Handling**: The codebase uses explicit error returns rather than panics - maintain this pattern +5. **Logging**: Use the `logger` package for all logging needs, avoid fmt.Print statements + +### Common Tasks + +To add a new utility module: +1. Create a new directory at the root level +2. Follow the existing pattern of having a main file with the package name +3. Include comprehensive tests in `*_test.go` files +4. Update imports in other packages if the utility is widely useful + +To modify the HTTP client: +1. Changes go in `http/client.go` or `http/middleware.go` +2. Authentication providers are in `http/auth_*.go` files +3. Always maintain backward compatibility with the builder pattern + +To work with the dependency installer: +1. Dependencies are defined in `deps/binary.go` +2. Binary definitions include version, download URLs, and install paths +3. Use templates for cross-platform path handling \ No newline at end of file diff --git a/collections/collections.go b/collections/collections.go index 646faf0..5b0dc57 100644 --- a/collections/collections.go +++ b/collections/collections.go @@ -1,3 +1,28 @@ +// Package collections provides generic utilities for working with +// collections like slices, maps, sets, and priority queues. +// +// The package leverages Go generics to provide type-safe operations +// without runtime overhead. It includes utilities for common operations +// like filtering, mapping, grouping, and set operations. +// +// Slice Operations: +// +// numbers := []int{1, 2, 3, 4, 5} +// evens := collections.Filter(numbers, func(n int) bool { return n%2 == 0 }) +// squares := collections.Map(numbers, func(n int) int { return n * n }) +// +// Map Operations: +// +// data := map[string]int{"a": 1, "b": 2} +// keys := collections.Keys(data) +// values := collections.Values(data) +// +// Set Operations: +// +// set1 := collections.NewSet(1, 2, 3) +// set2 := collections.NewSet(2, 3, 4) +// union := set1.Union(set2) // {1, 2, 3, 4} +// intersect := set1.Intersect(set2) // {2, 3} package collections import ( @@ -19,7 +44,15 @@ func takeSliceArg(arg interface{}) (out []interface{}, ok bool) { return out, true } -// ToString takes an object and tries to convert it to a string +// ToString converts various types to their string representation. +// Handles slices by joining elements with commas, fmt.Stringer types, +// strings, booleans, and falls back to fmt.Sprintf for other types. +// +// Example: +// +// collections.ToString([]int{1, 2, 3}) // "1, 2, 3" +// collections.ToString(true) // "true" +// collections.ToString(nil) // "" func ToString(i interface{}) string { if slice, ok := takeSliceArg(i); ok { s := "" diff --git a/collections/maps.go b/collections/maps.go index 855f0b6..9666f2c 100644 --- a/collections/maps.go +++ b/collections/maps.go @@ -8,7 +8,13 @@ import ( "strings" ) -// ToGenericMap converts a map[string]string to a map[string]interface{} +// ToGenericMap converts a map[string]string to a map[string]interface{}. +// Useful when you need to pass string maps to functions expecting generic maps. +// +// Example: +// +// strMap := map[string]string{"name": "John", "age": "30"} +// generic := collections.ToGenericMap(strMap) func ToGenericMap(m map[string]string) map[string]interface{} { var out = map[string]interface{}{} for k, v := range m { @@ -17,7 +23,14 @@ func ToGenericMap(m map[string]string) map[string]interface{} { return out } -// ToStringMap converts a map[string]interface{} to a map[string]string by using each objects String() fn +// ToStringMap converts a map[string]interface{} to a map[string]string. +// Each value is converted using fmt.Sprintf("%v", value). +// +// Example: +// +// data := map[string]interface{}{"count": 42, "active": true} +// strings := collections.ToStringMap(data) +// // Result: {"count": "42", "active": "true"} func ToStringMap(m map[string]interface{}) map[string]string { var out = make(map[string]string) for k, v := range m { @@ -26,7 +39,17 @@ func ToStringMap(m map[string]interface{}) map[string]string { return out } -// ToBase64Map converts a map[string]interface{} to a map[string]string by converting []byte to base64 +// ToBase64Map converts a map[string]interface{} to a map[string]string. +// []byte values are encoded as base64, other types use fmt.Sprintf. +// +// Example: +// +// data := map[string]interface{}{ +// "text": "hello", +// "binary": []byte{0x48, 0x65, 0x6c, 0x6c, 0x6f}, +// } +// encoded := collections.ToBase64Map(data) +// // Result: {"text": "hello", "binary": "SGVsbG8="} func ToBase64Map(m map[string]interface{}) map[string]string { var out = make(map[string]string) for k, v := range m { @@ -40,7 +63,8 @@ func ToBase64Map(m map[string]interface{}) map[string]string { return out } -// ToByteMap converts a map[string]interface{} to a map[string]string by converting []byte to base64 +// ToByteMap converts a map[string]interface{} to a map[string][]byte. +// String values are converted to []byte, []byte values are kept as-is. func ToByteMap(m map[string]interface{}) map[string][]byte { var out = make(map[string][]byte) for k, v := range m { @@ -54,8 +78,15 @@ func ToByteMap(m map[string]interface{}) map[string][]byte { return out } -// MergeMap will merge map b into a. -// On key collision, map b takes precedence. +// MergeMap merges two maps, with values from b taking precedence. +// Returns the merged map (modifies map a in place). +// +// Example: +// +// defaults := map[string]int{"timeout": 30, "retries": 3} +// overrides := map[string]int{"timeout": 60} +// config := collections.MergeMap(defaults, overrides) +// // Result: {"timeout": 60, "retries": 3} func MergeMap[T1 comparable, T2 any](a, b map[T1]T2) map[T1]T2 { if a == nil { a = make(map[T1]T2) @@ -72,10 +103,14 @@ func MergeMap[T1 comparable, T2 any](a, b map[T1]T2) map[T1]T2 { return a } -// KeyValueSliceToMap takes in a list of strings in a=b format -// and returns a map from it. +// KeyValueSliceToMap converts a slice of "key=value" strings to a map. +// Strings without '=' are treated as keys with empty values. +// +// Example: // -// Any string that's not in a=b format will be ignored. +// args := []string{"env=prod", "debug=true", "verbose"} +// config := collections.KeyValueSliceToMap(args) +// // Result: {"env": "prod", "debug": "true", "verbose": ""} func KeyValueSliceToMap(in []string) map[string]string { sanitized := make(map[string]string, len(in)) for _, item := range in { @@ -89,15 +124,27 @@ func KeyValueSliceToMap(in []string) map[string]string { return sanitized } -// SelectorToMap returns a map from a selector string. -// i.e. "a=b,c=d" -> {"a": "b", "c": "d"} +// SelectorToMap parses a comma-separated selector string into a map. +// Commonly used for Kubernetes-style label selectors. +// +// Example: +// +// selector := "app=nginx,env=prod,tier=frontend" +// labels := collections.SelectorToMap(selector) +// // Result: {"app": "nginx", "env": "prod", "tier": "frontend"} func SelectorToMap(selector string) (matchLabels map[string]string) { labels := strings.Split(selector, ",") return KeyValueSliceToMap(labels) } -// SortedMap takes a map and returns a sorted key value string -// i.e. {"a": "b", "c": "d"} -> "a=b,c=d" +// SortedMap converts a map to a sorted key=value string representation. +// Keys are sorted alphabetically for consistent output. +// +// Example: +// +// labels := map[string]string{"tier": "web", "app": "nginx"} +// result := collections.SortedMap(labels) +// // Result: "app=nginx,tier=web" func SortedMap(m map[string]string) string { keys := MapKeys(m) sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) @@ -110,7 +157,13 @@ func SortedMap(m map[string]string) string { return strings.Join(items, ",") } -// MapToIni takes a map and converts it into an INI formatted string +// MapToIni converts a map to INI format with one key=value pair per line. +// +// Example: +// +// config := map[string]string{"host": "localhost", "port": "8080"} +// ini := collections.MapToIni(config) +// // Result: "host=localhost\nport=8080\n" func MapToIni(Map map[string]string) string { str := "" for k, v := range Map { @@ -137,6 +190,13 @@ func IniToMap(path string) map[string]string { return result } +// MapKeys returns a slice containing all keys from the map. +// The order of keys is non-deterministic. +// +// Example: +// +// users := map[int]string{1: "Alice", 2: "Bob", 3: "Charlie"} +// ids := collections.MapKeys(users) // [1, 2, 3] (order may vary) func MapKeys[K comparable, T any](m map[K]T) []K { keys := make([]K, 0, len(m)) for k := range m { diff --git a/collections/priorityqueue.go b/collections/priorityqueue.go index f554b04..753ee93 100644 --- a/collections/priorityqueue.go +++ b/collections/priorityqueue.go @@ -2,17 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package priorityqueue implements a priority queue backed by binary queue. -// -// A thread-safe priority queue based on a priority queue. -// The elements of the priority queue are ordered by a comparator provided at queue construction time. -// -// The heap of this queue is the least/smallest element with respect to the specified ordering. -// If multiple elements are tied for least value, the heap is one of those elements arbitrarily. -// -// Structure is thread safe. -// -// References: https://en.wikipedia.org/wiki/Priority_queue package collections import ( @@ -182,6 +171,20 @@ type queueItem[T comparable] struct { var _ queues.Queue[int] = (*Queue[int])(nil) // Queue holds elements in an array-list +// Queue is a thread-safe priority queue implementation. +// Elements are ordered by a comparator function and can optionally +// be deduplicated using an equality function. +// +// Example: +// +// queue, _ := collections.NewQueue(collections.QueueOpts[int]{ +// Comparator: func(a, b int) int { return a - b }, // Min heap +// Dedupe: true, +// Equals: func(a, b int) bool { return a == b }, +// }) +// queue.Enqueue(5) +// queue.Enqueue(1) +// item, _ := queue.Dequeue() // Returns 1 type Queue[T comparable] struct { heap *binaryheap.Heap[queueItem[T]] Comparator utils.Comparator[T] @@ -191,6 +194,7 @@ type Queue[T comparable] struct { Dedupe bool } +// QueueOpts configures a priority queue. type QueueOpts[T comparable] struct { Comparator utils.Comparator[T] Dedupe bool @@ -198,6 +202,17 @@ type QueueOpts[T comparable] struct { Metrics MetricsOpts[T] } +// NewQueue creates a new priority queue with the specified options. +// Returns an error if Dedupe is true but no Equals function is provided. +// +// Example: +// +// // Create a min-heap priority queue for tasks +// queue, err := collections.NewQueue(collections.QueueOpts[Task]{ +// Comparator: func(a, b Task) int { +// return a.Priority - b.Priority // Lower priority first +// }, +// }) func NewQueue[T comparable](opts QueueOpts[T]) (*Queue[T], error) { if opts.Dedupe && opts.Equals == nil { return nil, errors.New("dedupe requires Equals function") diff --git a/collections/slice.go b/collections/slice.go index 2d44a76..058ecd9 100644 --- a/collections/slice.go +++ b/collections/slice.go @@ -6,6 +6,13 @@ import ( "strings" ) +// Dedup removes duplicate elements from a slice while preserving order. +// Returns a new slice containing only the first occurrence of each element. +// +// Example: +// +// nums := []int{1, 2, 2, 3, 1, 4} +// unique := collections.Dedup(nums) // [1, 2, 3, 4] func Dedup[T comparable](arr []T) []T { set := make(map[T]bool) retArr := []T{} @@ -18,7 +25,13 @@ func Dedup[T comparable](arr []T) []T { return retArr } -// ReplaceAllInSlice runs strings.Replace on all elements in a slice and returns the result +// ReplaceAllInSlice applies strings.ReplaceAll to each element in the slice. +// +// Example: +// +// urls := []string{"http://api.com", "http://web.com"} +// secure := collections.ReplaceAllInSlice(urls, "http://", "https://") +// // Result: ["https://api.com", "https://web.com"] func ReplaceAllInSlice(a []string, find string, replacement string) (replaced []string) { for _, s := range a { replaced = append(replaced, strings.ReplaceAll(s, find, replacement)) @@ -26,7 +39,13 @@ func ReplaceAllInSlice(a []string, find string, replacement string) (replaced [] return } -// SplitAllInSlice runs strings.Split on all elements in a slice and returns the results at the given index +// SplitAllInSlice splits each element and returns the part at the specified index. +// +// Example: +// +// emails := []string{"john@example.com", "jane@test.org"} +// domains := collections.SplitAllInSlice(emails, "@", 1) +// // Result: ["example.com", "test.org"] func SplitAllInSlice(a []string, split string, index int) (replaced []string) { for _, s := range a { replaced = append(replaced, strings.Split(s, split)[index]) @@ -34,8 +53,13 @@ func SplitAllInSlice(a []string, split string, index int) (replaced []string) { return } -// Find returns the smallest index i at which x == a[i], -// or len(a) if there is no such index. +// Find returns the index of the first occurrence of x in the slice, +// or len(a) if x is not found. +// +// Example: +// +// fruits := []string{"apple", "banana", "orange"} +// idx := collections.Find(fruits, "banana") // Returns 1 func Find(a []string, x string) int { for i, n := range a { if x == n { @@ -45,7 +69,14 @@ func Find(a []string, x string) int { return len(a) } -// Contains tells whether a contains x. +// Contains checks if a slice contains the specified element. +// +// Example: +// +// nums := []int{1, 2, 3, 4, 5} +// if collections.Contains(nums, 3) { +// // Element found +// } func Contains[T comparable](a []T, x T) bool { for _, n := range a { if x == n { diff --git a/console/console.go b/console/console.go index 278fac5..1cd87fa 100644 --- a/console/console.go +++ b/console/console.go @@ -41,7 +41,7 @@ var ( LightCyan = cyan + Light BrightCyan = cyan + Bright white = "\x1b[38" - White = white + Light + White = white BoldOn = "\x1b[1m" BoldOff = "\x1b[22m" DarkWhite = "\x1b[38;5;244m" diff --git a/context/context.go b/context/context.go index 6a804c9..ea7d750 100644 --- a/context/context.go +++ b/context/context.go @@ -1,3 +1,36 @@ +// Package context provides an enhanced context implementation that combines +// standard Go context functionality with integrated logging and distributed tracing. +// +// The Context type embeds context.Context and adds: +// - Integrated structured logging with level control +// - OpenTelemetry tracing support +// - Debug and trace mode flags +// - Automatic correlation between logs and traces +// +// Basic Usage: +// +// ctx := context.NewContext(context.Background()) +// ctx.Infof("Processing request %s", requestID) +// +// // Enable debug logging +// debugCtx := ctx.WithDebug() +// debugCtx.Debugf("Detailed information: %v", details) +// +// // Start a traced operation +// ctx, span := ctx.StartSpan("database-query") +// defer span.End() +// ctx.Infof("Executing query: %s", query) +// +// With Custom Logger and Tracer: +// +// ctx := context.NewContext( +// context.Background(), +// context.WithLogger(customLogger), +// context.WithTracer(otel.Tracer("my-service")), +// ) +// +// The context automatically propagates logging configuration and trace spans +// through the call chain, ensuring consistent observability across your application. package context import ( @@ -20,32 +53,71 @@ var ( noopTracer = noop.NewTracerProvider().Tracer("noop") ) +// ContextOptions is a function that configures a Context during creation. type ContextOptions func(*Context) +// WithTraceFn sets a custom function to determine if trace logging is enabled. +// This allows dynamic control of trace logging based on context values or external conditions. +// +// Example: +// +// ctx := NewContext(context.Background(), WithTraceFn(func(ctx Context) *bool { +// // Enable trace for specific user IDs +// userID := ctx.Value("userID") +// enabled := userID == "debug-user" +// return &enabled +// })) func WithTraceFn(fn func(Context) *bool) ContextOptions { return func(opts *Context) { opts.isTraceFn = fn } } +// WithDebugFn sets a custom function to determine if debug logging is enabled. +// This allows dynamic control of debug logging based on context values or external conditions. func WithDebugFn(fn func(Context) *bool) ContextOptions { return func(opts *Context) { opts.isDebugFn = fn } } +// WithTracer configures the OpenTelemetry tracer for the context. +// If not specified, a no-op tracer is used. +// +// Example: +// +// tracer := otel.Tracer("my-service") +// ctx := NewContext(context.Background(), WithTracer(tracer)) func WithTracer(tracer trace.Tracer) ContextOptions { return func(opts *Context) { opts.tracer = tracer } } +// WithLogger sets a custom logger for the context. +// If not specified, the standard logger is used. func WithLogger(log logger.Logger) ContextOptions { return func(opts *Context) { opts.Logger = log } } +// NewContext creates a new enhanced context from a standard Go context. +// The context is configured with the provided options and defaults to +// the standard logger and a no-op tracer if not specified. +// +// Example: +// +// // Basic context with defaults +// ctx := NewContext(context.Background()) +// +// // Context with custom configuration +// ctx := NewContext( +// context.Background(), +// WithLogger(myLogger), +// WithTracer(myTracer), +// WithDebugFn(customDebugCheck), +// ) func NewContext(basectx gocontext.Context, opts ...ContextOptions) Context { ctx := Context{ Context: basectx, @@ -63,6 +135,17 @@ func NewContext(basectx gocontext.Context, opts ...ContextOptions) Context { return ctx } +// Context is an enhanced context that embeds the standard context.Context +// and adds integrated logging and tracing capabilities. +// +// It provides: +// - Structured logging with automatic trace correlation +// - Debug and trace mode management +// - OpenTelemetry span creation and management +// - Context value propagation with type safety +// +// The Context maintains its configuration (logger, tracer, debug/trace settings) +// when creating derived contexts through WithValue, WithTimeout, etc. type Context struct { gocontext.Context Logger logger.Logger diff --git a/dns/dns.go b/dns/dns.go index 0eb5fda..a0f3f98 100644 --- a/dns/dns.go +++ b/dns/dns.go @@ -1,3 +1,33 @@ +// Package dns provides DNS lookup functionality with built-in caching support. +// +// The package offers a simple API for resolving hostnames to IP addresses +// with automatic caching to improve performance and reduce DNS query load. +// It handles both raw hostnames and URLs, automatically extracting the +// hostname portion when needed. +// +// Features: +// - Automatic caching with 1-hour TTL +// - Support for both hostnames and URLs +// - IPv4 address filtering +// - Direct IP address passthrough +// +// Basic Usage: +// +// // Lookup with caching (recommended) +// ips, err := dns.CacheLookup("A", "example.com") +// if err != nil { +// log.Fatal(err) +// } +// for _, ip := range ips { +// fmt.Printf("IP: %s\n", ip) +// } +// +// // Direct lookup without caching +// ips, err := dns.Lookup("A", "https://example.com") +// // Automatically extracts hostname from URL +// +// The package prioritizes IPv4 addresses in results and handles IP addresses +// directly without performing DNS lookups. package dns import ( @@ -12,6 +42,24 @@ import ( // dnsCache serves as the global cache for dns queries var dnsCache = cache.New(time.Hour, time.Hour) +// CacheLookup performs a DNS lookup with caching support. +// Results are cached for 1 hour to reduce DNS query load. +// +// Parameters: +// - recordType: DNS record type (e.g., "A", "AAAA"). Currently not used but reserved for future use. +// - hostname: The hostname, URL, or IP address to resolve +// +// Returns: +// - []net.IP: Slice of resolved IPv4 addresses +// - error: If the lookup fails or hostname is invalid +// +// Example: +// +// ips, err := CacheLookup("A", "google.com") +// if err != nil { +// return err +// } +// fmt.Printf("Resolved IPs: %v\n", ips) func CacheLookup(recordType, hostname string) ([]net.IP, error) { key := fmt.Sprintf("%s:%s", recordType, hostname) @@ -30,7 +78,23 @@ func CacheLookup(recordType, hostname string) ([]net.IP, error) { return ips, nil } -// Lookup looks up the record using Go's default resolver +// Lookup performs a direct DNS lookup without caching. +// It uses Go's default resolver to query DNS servers. +// +// The function handles multiple input formats: +// - Raw hostnames: "example.com" +// - URLs: "https://example.com:8080/path" (extracts hostname) +// - IP addresses: "192.168.1.1" (returns immediately without lookup) +// +// Only IPv4 addresses are returned in the result. +// +// Parameters: +// - recordType: DNS record type (currently not used but reserved for future use) +// - hostname: The hostname, URL, or IP address to resolve +// +// Returns: +// - []net.IP: Slice of resolved IPv4 addresses +// - error: If the lookup fails or hostname is invalid func Lookup(recordType, hostname string) ([]net.IP, error) { host := hostname if url, err := url.Parse(hostname); err != nil { diff --git a/duration/duration.go b/duration/duration.go index 21828ab..dc8ce81 100644 --- a/duration/duration.go +++ b/duration/duration.go @@ -9,7 +9,53 @@ // FORKED from https://git.maze.io/go/duration/src/branch/master/duration.go -// Package duration contains routines to parse standard units of time. +// Package duration provides enhanced duration parsing and formatting with support +// for extended time units beyond the standard library. +// +// This package extends Go's standard time.Duration to support larger units like +// days, weeks, months, and years, making it more suitable for human-readable +// time representations and configuration files. +// +// Key features: +// - Parse durations with extended units: "d" (days), "w" (weeks), "y" (years) +// - Human-readable string formatting: "3d12h" instead of "84h" +// - Automatic precision adjustment based on magnitude +// - Compatible with standard time.Duration +// +// Supported time units: +// - ns: nanoseconds +// - us/µs/μs: microseconds +// - ms: milliseconds +// - s: seconds +// - m: minutes +// - h: hours +// - d: days (24 hours) +// - w: weeks (7 days) +// - y: years (365 days, approximation) +// +// Basic usage: +// +// // Parse duration strings +// d, err := duration.ParseDuration("3d12h30m") +// if err != nil { +// log.Fatal(err) +// } +// fmt.Println(d) // Output: 3d12h30m +// +// // Create from standard units +// d = duration.Day * 7 + duration.Hour * 6 +// fmt.Println(d) // Output: 1w6h +// +// // Convert to standard time.Duration +// td := time.Duration(d) +// +// // Get components +// fmt.Printf("Hours: %.2f, Days: %.2f\n", d.Hours(), d.Days()) +// +// The formatting automatically adjusts precision based on the duration magnitude: +// - Durations over a week: formatted as weeks, days, hours (1w2d3h) +// - Durations over a day: formatted as days, hours, minutes (2d3h45m) +// - Smaller durations: include seconds and sub-second precision package duration import ( @@ -18,7 +64,9 @@ import ( "time" ) -// Duration is a standard unit of time. +// Duration represents a time duration with support for extended units. +// It embeds time.Duration and can be used anywhere a time.Duration is expected, +// but provides enhanced parsing and formatting capabilities. type Duration time.Duration // String returns a string representing the duration in the form "3d1h3m". @@ -301,11 +349,28 @@ var unitMap = map[string]int64{ "y": int64(Year), // Approximation } -// ParseDuration parses a duration string. -// A duration string is a possibly signed sequence of -// decimal numbers, each with optional fraction and a unit suffix, -// such as "300ms", "-1.5h" or "2h45m". -// Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h", "d", "w", "y". +// ParseDuration parses a duration string with support for extended time units. +// A duration string is a possibly signed sequence of decimal numbers, each with +// optional fraction and a unit suffix, such as "300ms", "-1.5h", "2h45m", or "3d12h". +// +// Valid time units are: +// - "ns": nanoseconds +// - "us" (or "µs"/"μs"): microseconds +// - "ms": milliseconds +// - "s": seconds +// - "m": minutes +// - "h": hours +// - "d": days (24 hours) +// - "w": weeks (7 days) +// - "y": years (365 days) +// +// Examples: +// +// ParseDuration("2h30m") // 2 hours 30 minutes +// ParseDuration("1.5d") // 1.5 days (36 hours) +// ParseDuration("3w2d") // 3 weeks 2 days +// ParseDuration("-30m") // negative 30 minutes +// ParseDuration("1d12h30m") // 1 day 12 hours 30 minutes func ParseDuration(s string) (Duration, error) { // [-+]?([0-9]*(\.[0-9]*)?[a-z]+)+ orig := s diff --git a/hash/hash.go b/hash/hash.go index b95ca10..cdfa522 100644 --- a/hash/hash.go +++ b/hash/hash.go @@ -1,3 +1,42 @@ +// Package hash provides utility functions for generating cryptographic hashes +// and deterministic UUIDs from various data types. +// +// The package offers convenient methods to generate hashes from JSON-serializable +// objects and create deterministic UUIDs based on input data, which is useful +// for generating stable identifiers across distributed systems. +// +// Key features: +// - JSON object hashing with MD5 +// - SHA256 string hashing +// - Deterministic UUID generation from any JSON-serializable data +// +// Basic usage: +// +// // Hash a struct to MD5 +// type User struct { +// Name string +// Email string +// } +// user := User{Name: "John", Email: "john@example.com"} +// hash, err := hash.JSONMD5Hash(user) +// if err != nil { +// log.Fatal(err) +// } +// fmt.Printf("MD5: %s\n", hash) +// +// // Generate SHA256 hash +// sha := hash.Sha256Hex("hello world") +// fmt.Printf("SHA256: %s\n", sha) +// +// // Create deterministic UUID +// id, err := hash.DeterministicUUID(user) +// if err != nil { +// log.Fatal(err) +// } +// fmt.Printf("UUID: %s\n", id) +// +// Note: MD5 is used for non-cryptographic purposes like checksums and +// deterministic ID generation. For cryptographic security, use SHA256 or stronger. package hash import ( @@ -9,8 +48,21 @@ import ( "github.com/google/uuid" ) -// JSONMD5Hash marshals the object into JSON and generates its md5 hash -// in hex format. +// JSONMD5Hash marshals the object into JSON and generates its MD5 hash +// in hexadecimal format. This is useful for creating checksums of +// structured data or generating cache keys. +// +// The object must be JSON-serializable. The resulting hash is deterministic +// for the same input data, making it suitable for comparison and deduplication. +// +// Note: MD5 is not cryptographically secure and should only be used for +// non-security purposes like checksums and identifiers. +// +// Example: +// +// config := map[string]string{"host": "localhost", "port": "8080"} +// hash, err := JSONMD5Hash(config) +// // hash will be consistent for the same config values func JSONMD5Hash(obj any) (string, error) { data, err := json.Marshal(obj) if err != nil { @@ -25,6 +77,15 @@ func JSONMD5Hash(obj any) (string, error) { return hex.EncodeToString(hash[:]), nil } +// Sha256Hex computes the SHA256 hash of the input string and returns it +// as a hexadecimal string. SHA256 is cryptographically secure and suitable +// for security-sensitive applications. +// +// Example: +// +// password := "secretpassword" +// hash := Sha256Hex(password) +// // hash = "5e884898da28047151d0e56f8dc6292773603d0d6aabbdd62a11ef721d1542d8" func Sha256Hex(in string) string { hash := sha256.New() hash.Write([]byte(in)) @@ -32,6 +93,25 @@ func Sha256Hex(in string) string { return hex.EncodeToString(hashVal[:]) } +// DeterministicUUID generates a UUID that is deterministic based on the input seed. +// The same seed will always produce the same UUID, which is useful for creating +// stable identifiers in distributed systems. +// +// The seed can be any JSON-serializable object. The function uses MD5 hashing +// internally to generate a 128-bit value that forms the UUID. +// +// Example: +// +// // Generate UUID from user data +// userData := map[string]string{ +// "email": "user@example.com", +// "system": "production", +// } +// id, err := DeterministicUUID(userData) +// // The same userData will always generate the same UUID +// +// // Generate UUID from string +// id2, err := DeterministicUUID("unique-resource-name") func DeterministicUUID(seed any) (uuid.UUID, error) { byteHash, err := JSONMD5Hash(seed) if err != nil { diff --git a/http/client.go b/http/client.go index 4e9b723..ec26549 100644 --- a/http/client.go +++ b/http/client.go @@ -1,3 +1,37 @@ +// Package http provides an enhanced HTTP client with built-in support for +// authentication, retries, tracing, and middleware. +// +// The client supports multiple authentication methods (Basic, Digest, NTLM, OAuth), +// automatic retries with exponential backoff, request/response tracing, and a +// flexible middleware system. +// +// Basic Usage: +// +// client := http.NewClient() +// resp, err := client.R(context.Background()).GET("https://api.example.com/data") +// +// With Authentication: +// +// client := http.NewClient(). +// Auth("username", "password"). +// Digest(true) +// +// With Retries and Timeout: +// +// client := http.NewClient(). +// Retry(3, time.Second, 2.0). +// Timeout(30 * time.Second) +// +// With Request Tracing: +// +// client := http.NewClient(). +// Trace(http.TraceAll) // Log full request/response details +// +// The client can also be used as a standard http.RoundTripper: +// +// httpClient := &http.Client{ +// Transport: http.NewClient().Auth("user", "pass").RoundTripper(), +// } package http import ( @@ -13,6 +47,7 @@ import ( dac "github.com/Snawoot/go-http-digest-auth-client" "github.com/flanksource/commons/dns" "github.com/flanksource/commons/http/middlewares" + "github.com/flanksource/commons/logger" httpntlm "github.com/vadimi/go-http-ntlm" httpntlmv2 "github.com/vadimi/go-http-ntlm/v2" ) @@ -65,7 +100,20 @@ type AuthConfig struct { Ntlmv2 bool } -// Client is a type that represents an HTTP client +// Client is an enhanced HTTP client with built-in support for authentication, +// retries, tracing, and middleware. It provides a fluent API for configuring +// requests and can be used as a drop-in replacement for http.Client. +// +// Example: +// +// client := http.NewClient(). +// Auth("user", "pass"). +// Retry(3, time.Second, 2.0). +// Trace(http.TraceHeaders). +// Timeout(30 * time.Second) +// +// resp, err := client.R(ctx). +// GET("https://api.example.com/data") type Client struct { httpClient *http.Client @@ -99,7 +147,37 @@ type Client struct { tlsConfig *tls.Config } -// NewClient configures a new HTTP client using given configuration +// RoundTrip implements http.RoundTripper. +func (c *Client) RoundTrip(r *http.Request) (*http.Response, error) { + // Convert http.Request to our custom Request type + req := &Request{ + ctx: r.Context(), + client: c, + method: r.Method, + url: r.URL, + body: r.Body, + headers: r.Header, + queryParams: r.URL.Query(), + retryConfig: c.retryConfig, + } + + resp, err := c.roundTrip(req) + if resp == nil { + return nil, err + } + return resp.Response, err +} + +// NewClient creates a new HTTP client with default settings. +// The client has a default timeout of 2 minutes and can be customized +// using the fluent API methods. +// +// Example: +// +// client := http.NewClient(). +// BaseURL("https://api.example.com"). +// Header("X-API-Key", "secret"). +// InsecureSkipVerify(true) func NewClient() *Client { client := &http.Client{ Timeout: time.Minute * 2, @@ -112,7 +190,16 @@ func NewClient() *Client { } } -// R create a new request. +// R creates a new request with the given context. +// The request inherits all settings from the client (headers, auth, etc.) +// and can be further customized before execution. +// +// Example: +// +// resp, err := client.R(ctx). +// Header("X-Request-ID", "123"). +// QueryParam("page", "1"). +// GET("/users") func (c *Client) R(ctx context.Context) *Request { return &Request{ ctx: ctx, @@ -128,9 +215,19 @@ func (c *Client) UserAgent(agent string) *Client { return c } -// Retry configuration retrying on failure with exponential backoff. +// Retry configures automatic retry behavior with exponential backoff. +// Failed requests will be retried up to maxRetries times, with delays +// calculated as baseDuration * (exponent ^ attemptNumber). +// +// Parameters: +// - maxRetries: Maximum number of retry attempts (0 disables retries) +// - baseDuration: Initial delay between retries +// - exponent: Multiplier for exponential backoff (typically 2.0) +// +// Example: // -// Base duration of a second & an exponent of 2 is a good option. +// // Retry up to 3 times with delays of 1s, 2s, 4s +// client.Retry(3, time.Second, 2.0) func (c *Client) Retry(maxRetries uint, baseDuration time.Duration, exponent float64) *Client { c.retryConfig.MaxRetries = maxRetries c.retryConfig.RetryWait = baseDuration @@ -148,8 +245,17 @@ func (c *Client) Header(key, val string) *Client { return c } -// ConnectTo specifies the host:port on which the URL is sought. -// If empty, the URL's host is used. +// ConnectTo overrides the target host:port for requests. +// This is useful for testing against specific IPs or when dealing with +// DNS issues. The original hostname is preserved in the Host header. +// +// Example: +// +// // Connect to a specific IP instead of using DNS +// client.ConnectTo("192.168.1.100:8080") +// +// // Test against localhost while preserving the original Host header +// client.ConnectTo("localhost:3000") func (c *Client) ConnectTo(host string) *Client { c.connectTo = host return c @@ -205,6 +311,18 @@ type TLSConfig struct { Key string } +// TLSConfig configures advanced TLS settings including custom CAs, +// client certificates, and handshake timeout. +// +// Example: +// +// client.TLSConfig(TLSConfig{ +// CA: caPEM, // Custom CA certificate +// Cert: clientCert, // Client certificate for mTLS +// Key: clientKey, // Client private key +// InsecureSkipVerify: false, // Verify server certificate +// HandshakeTimeout: 10 * time.Second, +// }) func (c *Client) TLSConfig(conf TLSConfig) (*Client, error) { c.initTLSConfig() @@ -241,8 +359,14 @@ func (c *Client) TLSConfig(conf TLSConfig) (*Client, error) { return c, nil } -// InsecureSkipVerify controls whether a client verifies the server's -// certificate chain and host name +// InsecureSkipVerify disables TLS certificate verification. +// WARNING: This makes the client vulnerable to man-in-the-middle attacks. +// Only use in development or when connecting to services with self-signed certificates. +// +// Example: +// +// // Accept any certificate (dangerous!) +// client.InsecureSkipVerify(true) func (c *Client) InsecureSkipVerify(val bool) *Client { c.initTLSConfig() @@ -268,7 +392,20 @@ func (c *Client) setProxy(proxyURL *url.URL) { c.httpClient.Transport = customTransport } -// Auth sets up the username & password for basic auth or NTLM. +// Auth configures authentication credentials for the client. +// By default, this sets up HTTP Basic Authentication. +// Use Digest(), NTLM(), or NTLMV2() to switch authentication methods. +// +// Example: +// +// // Basic auth +// client.Auth("username", "password") +// +// // Digest auth +// client.Auth("username", "password").Digest(true) +// +// // NTLM auth with domain +// client.Auth("DOMAIN\\username", "password").NTLM(true) func (c *Client) Auth(username, password string) *Client { if c.authConfig == nil { c.authConfig = &AuthConfig{} @@ -279,11 +416,38 @@ func (c *Client) Auth(username, password string) *Client { return c } +// OAuth configures OAuth 2.0 authentication for the client. +// Supports various OAuth flows including client credentials and authorization code. +// +// Example: +// +// client.OAuth(middlewares.OauthConfig{ +// ClientID: "client-id", +// ClientSecret: "client-secret", +// TokenURL: "https://auth.example.com/token", +// Scopes: []string{"read", "write"}, +// }) func (c *Client) OAuth(config middlewares.OauthConfig) *Client { c.Use(middlewares.NewOauthTransport(config).RoundTripper) return c } +// Trace enables request/response tracing with customizable detail levels. +// Traced information is added to the context and can be retrieved by middleware. +// +// Use predefined configs for common scenarios: +// - TraceAll: Full details including headers, body, and TLS +// - TraceHeaders: Headers and query params only (no body) +// +// Example: +// +// client.Trace(http.TraceConfig{ +// Headers: true, +// ResponseHeaders: true, +// Body: true, +// Response: true, +// MaxBodyLength: 1024, // Limit body size in traces +// }) func (c *Client) Trace(config TraceConfig) *Client { c.Use(middlewares.NewTracedTransport(config).RoundTripper) return c @@ -294,6 +458,27 @@ func (c *Client) TraceToStdout(config TraceConfig) *Client { return c } +// WithHttpLogging enables HTTP request/response logging based on the provided log levels. +// +// Parameters: +// - headerLevel: The minimum log level required to log HTTP headers (e.g., logger.Debug) +// - bodyLevel: The minimum log level required to log request/response bodies (e.g., logger.Trace) +// +// Example: +// client.WithHttpLogging(logger.Debug, logger.Trace) +// +// This will log headers when debug logging is enabled (-v or -v 1) and +// bodies when trace logging is enabled (-vv or -v 2 or higher). +// +// Note: When using with cobra commands, ensure UseCobraFlags is called +// in PersistentPreRun to properly parse -v N syntax. +func (c *Client) WithHttpLogging(headerLevel, bodyLevel logger.LogLevel) *Client { + c.Use(func(rt http.RoundTripper) http.RoundTripper { + return logger.NewHttpLoggerWithLevels(logger.GetLogger(), rt, headerLevel, bodyLevel) + }) + return c +} + func (c *Client) Digest(val bool) *Client { if c.authConfig == nil { c.authConfig = &AuthConfig{} @@ -303,6 +488,10 @@ func (c *Client) Digest(val bool) *Client { return c } +func (c *Client) RoundTripper() http.RoundTripper { + return c +} + func (c *Client) NTLM(val bool) *Client { if c.authConfig == nil { c.authConfig = &AuthConfig{} @@ -441,7 +630,20 @@ func toMap(h http.Header) map[string]string { return m } -// Use adds middleware to the client that wraps the client's transport +// Use adds middleware to the client's transport chain. +// Middleware functions wrap the underlying RoundTripper and can modify +// requests/responses, add logging, implement caching, etc. +// +// Middleware is applied in the order it was added. +// +// Example: +// +// client.Use(func(rt http.RoundTripper) http.RoundTripper { +// return middlewares.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { +// req.Header.Set("X-Custom", "value") +// return rt.RoundTrip(req) +// }) +// }) func (c *Client) Use(middlewares ...middlewares.Middleware) *Client { c.transportMiddlewares = append(c.transportMiddlewares, middlewares...) return c diff --git a/http/request.go b/http/request.go index 26bd6d3..ca1ed2e 100644 --- a/http/request.go +++ b/http/request.go @@ -12,6 +12,9 @@ import ( "time" ) +// Request represents an HTTP request that can be customized and executed. +// It provides a fluent API for setting headers, query parameters, body, and other options. +// Request instances should be created using Client.R(ctx). type Request struct { ctx context.Context client *Client @@ -36,28 +39,71 @@ func (r *Request) GetHeader(key string) string { return r.headers.Get(key) } -// Header set a header for the request. +// Header sets a header for the request. +// Multiple calls with the same key will overwrite previous values. +// +// Example: +// +// resp, err := client.R(ctx). +// Header("Authorization", "Bearer token"). +// Header("Content-Type", "application/json"). +// GET("/api/data") func (r *Request) Header(key, value string) *Request { r.headers.Set(key, value) return r } -// QueryParam sets query params +// QueryParam sets a query parameter for the request. +// Multiple calls with the same key will overwrite previous values. +// +// Example: +// +// resp, err := client.R(ctx). +// QueryParam("page", "1"). +// QueryParam("limit", "10"). +// GET("/api/users") func (r *Request) QueryParam(key, value string) *Request { r.queryParams.Set(key, value) return r } -// QueryParamAdd adds value to query param key +// QueryParamAdd adds a value to a query parameter. +// Unlike QueryParam, this allows multiple values for the same key. +// +// Example: +// +// // Results in: /api/search?tag=go&tag=http +// resp, err := client.R(ctx). +// QueryParamAdd("tag", "go"). +// QueryParamAdd("tag", "http"). +// GET("/api/search") func (r *Request) QueryParamAdd(key, value string) *Request { r.queryParams.Add(key, value) return r } +// Get performs an HTTP GET request to the specified URL. +// +// Example: +// +// resp, err := client.R(ctx).Get("https://api.example.com/users") +// if err != nil { +// return err +// } +// defer resp.Body.Close() func (r *Request) Get(url string) (*Response, error) { return r.Do(http.MethodGet, url) } +// Post performs an HTTP POST request with the given body. +// The body can be a string, []byte, io.Reader, or any type that can be JSON marshaled. +// +// Example: +// +// user := map[string]string{"name": "John", "email": "john@example.com"} +// resp, err := client.R(ctx). +// Header("Content-Type", "application/json"). +// Post("https://api.example.com/users", user) func (r *Request) Post(url string, body any) (*Response, error) { if err := r.Body(body); err != nil { return nil, fmt.Errorf("error setting body: %w", err) @@ -83,9 +129,20 @@ func (r *Request) Delete(url string) (*Response, error) { return r.Do(http.MethodDelete, url) } -// Retry configuration retrying on failure with exponential backoff. +// Retry configures retry behavior for this specific request, +// overriding the client's default retry configuration. +// +// Parameters: +// - maxRetries: Maximum number of retry attempts +// - baseDuration: Initial delay between retries +// - exponent: Multiplier for exponential backoff +// +// Example: // -// Base duration of a second & an exponent of 2 is a good option. +// // Retry this request up to 5 times +// resp, err := client.R(ctx). +// Retry(5, 500*time.Millisecond, 2.0). +// GET("/flaky-endpoint") func (r *Request) Retry(maxRetries uint, baseDuration time.Duration, exponent float64) *Request { r.retryConfig.MaxRetries = maxRetries r.retryConfig.RetryWait = baseDuration @@ -93,7 +150,22 @@ func (r *Request) Retry(maxRetries uint, baseDuration time.Duration, exponent fl return r } -// Body sets the request body +// Body sets the request body. Accepts multiple types: +// - io.Reader: Used directly as the body +// - []byte: Wrapped in a bytes.Reader +// - string: Converted to []byte and wrapped +// - Any other type: JSON marshaled +// +// Example: +// +// // JSON body +// req.Body(map[string]string{"key": "value"}) +// +// // String body +// req.Body("raw text data") +// +// // Reader body +// req.Body(bytes.NewReader(data)) func (r *Request) Body(v any) error { switch t := v.(type) { case io.Reader: diff --git a/logger/default.go b/logger/default.go index d9204df..13f4233 100644 --- a/logger/default.go +++ b/logger/default.go @@ -74,63 +74,122 @@ func BindFlags(flags *pflag.FlagSet) { flags.Bool("log-to-stderr", false, "Log to stderr instead of stdout") } +// UseCobraFlags initializes the logger using values from parsed cobra flags. +// This should be called after cobra has parsed the command line arguments. +func UseCobraFlags(flags *pflag.FlagSet) { + // Get the verbosity count from the parsed flags + if v, err := flags.GetCount("loglevel"); err == nil && v > 0 { + currentLogger.SetLogLevel(v) + } else if level, err := flags.GetString("log-level"); err == nil && level != "" && level != "info" { + currentLogger.SetLogLevel(level) + } + + // Apply other flags + if jsonLogs, err := flags.GetBool("json-logs"); err == nil && jsonLogs { + currentLogger = New("") + } +} + +// Warnf logs a warning message with formatting support. +// These are messages about potentially harmful situations. func Warnf(format string, args ...interface{}) { currentLogger.Warnf(format, args...) } +// Infof logs an informational message with formatting support. +// These are general informational messages about normal operations. func Infof(format string, args ...interface{}) { currentLogger.Infof(format, args...) } -// Secretf is like Tracef, but attempts to strip any secrets from the text +// Secretf logs a trace message after attempting to strip any secrets from the text. +// It automatically redacts common secret patterns like passwords, tokens, and API keys. +// +// Example: +// logger.Secretf("Connecting with password=%s", password) // password will be redacted func Secretf(format string, args ...interface{}) { currentLogger.Tracef(StripSecrets(fmt.Sprintf(format, args...))) } -// Prettyf is like Tracef, but pretty prints the entire struct +// Prettyf logs a trace message with a pretty-printed representation of the given object. +// Useful for debugging complex data structures. +// +// Example: +// logger.Prettyf("User data:", userStruct) // Logs formatted struct func Prettyf(msg string, obj interface{}) { currentLogger.Tracef(msg, Pretty(obj)) } +// Errorf logs an error message with formatting support. +// Use this for errors that need attention but don't terminate the program. func Errorf(format string, args ...interface{}) { currentLogger.Errorf(format, args...) } +// Debugf logs a debug message with formatting support. +// These messages are only shown when debug logging is enabled. func Debugf(format string, args ...interface{}) { currentLogger.Debugf(format, args...) } +// Tracef logs a trace message with formatting support. +// These are very detailed messages for troubleshooting, only shown at trace level. func Tracef(format string, args ...interface{}) { currentLogger.Tracef(format, args...) } +// Fatalf logs a fatal error message and terminates the program. +// Use this for unrecoverable errors. func Fatalf(format string, args ...interface{}) { currentLogger.Fatalf(format, args...) } +// V returns a verbose logger for conditional logging at the specified level. +// The level can be an integer or a LogLevel constant. +// +// Example: +// logger.V(2).Infof("Detailed info") // Only logs at verbosity 2+ func V(level any) Verbose { return currentLogger.V(level) } +// IsTraceEnabled returns true if trace level logging is enabled. func IsTraceEnabled() bool { return currentLogger.IsTraceEnabled() } +// IsLevelEnabled returns true if the specified verbosity level is enabled. +// +// Example: +// if logger.IsLevelEnabled(3) { +// // Perform expensive operation only if logging at level 3 +// } func IsLevelEnabled(level int) bool { return currentLogger.V(level).Enabled() } +// IsDebugEnabled returns true if debug level logging is enabled. func IsDebugEnabled() bool { return currentLogger.IsDebugEnabled() } +// WithValues returns a new logger with additional key-value pairs. +// These values will be included in all log messages from the returned logger. +// +// Example: +// userLogger := logger.WithValues("user_id", 123, "session", "abc") +// userLogger.Infof("User action") // Logs with user_id=123 session=abc func WithValues(keysAndValues ...interface{}) Logger { return currentLogger.WithValues(keysAndValues...) } +// SetLogger replaces the global logger instance. +// Use this to configure a custom logger implementation. func SetLogger(logger Logger) { currentLogger = logger } +// StandardLogger returns the current global logger instance. +// This is equivalent to GetLogger(). func StandardLogger() Logger { return currentLogger } diff --git a/logger/http.go b/logger/http.go index c20c5bb..dfc8299 100644 --- a/logger/http.go +++ b/logger/http.go @@ -14,6 +14,13 @@ var SensitiveHeaders = []string{ "Cookie", } +// NewHttpLogger creates an HTTP logger that logs at predefined levels. +// Deprecated: Use NewHttpLoggerWithLevels for more control over logging levels. +// +// Default behavior: +// - Headers and timing: Requires log level 5 (Trace3) +// - Request body: Requires log level 6 (Trace4) +// - Response body: Requires log level 7 func NewHttpLogger(logger Logger, rt http.RoundTripper) http.RoundTripper { if !logger.IsLevelEnabled(5) { return rt @@ -34,3 +41,35 @@ func NewHttpLogger(logger Logger, rt http.RoundTripper) http.RoundTripper { return l.RoundTripper(rt) } + +// NewHttpLoggerWithLevels creates an HTTP logger with configurable log levels for headers and body. +// +// Parameters: +// - logger: The logger instance to use +// - rt: The underlying RoundTripper to wrap +// - headerLevel: Minimum log level required to log headers, timing, and TLS info +// - bodyLevel: Minimum log level required to log request/response bodies +// +// Example: +// // Log headers at debug level (-v) and bodies at trace level (-vv) +// transport := NewHttpLoggerWithLevels(logger, http.DefaultTransport, logger.Debug, logger.Trace) +func NewHttpLoggerWithLevels(logger Logger, rt http.RoundTripper, headerLevel, bodyLevel LogLevel) http.RoundTripper { + if !logger.IsLevelEnabled(headerLevel) { + return rt + } + + l := &httpretty.Logger{ + Time: logger.IsLevelEnabled(headerLevel), + TLS: logger.IsLevelEnabled(headerLevel), + RequestHeader: logger.IsLevelEnabled(headerLevel), + RequestBody: logger.IsLevelEnabled(bodyLevel), + ResponseHeader: logger.IsLevelEnabled(headerLevel), + ResponseBody: logger.IsLevelEnabled(bodyLevel), + Colors: true, // erase line if you don't like colors + Formatters: []httpretty.Formatter{&httpretty.JSONFormatter{}}, + } + + l.SkipHeader(SensitiveHeaders) + + return l.RoundTripper(rt) +} diff --git a/logger/log.go b/logger/log.go index 6ad2d38..3d16fec 100644 --- a/logger/log.go +++ b/logger/log.go @@ -1,3 +1,32 @@ +// Package logger provides a flexible logging interface with support for +// multiple backends (logrus, slog) and various output formats. +// +// The package offers a global logger instance that can be configured with +// different log levels, output formats, and additional context values. +// +// Basic Usage: +// +// logger.Infof("Server started on port %d", 8080) +// logger.Debugf("Processing request: %s", requestID) +// logger.Errorf("Failed to connect to database: %v", err) +// +// With Context Values: +// +// log := logger.GetLogger().WithValues("user", userID, "request", requestID) +// log.Infof("Processing user request") +// +// Named Loggers: +// +// dbLogger := logger.GetLogger().Named("database") +// apiLogger := logger.GetLogger().Named("api") +// +// Log Levels: +// +// logger.SetLogLevel(logger.Debug) // Enable debug logging +// logger.SetLogLevel(logger.Trace) // Enable trace logging +// +// The package supports standard log levels (Info, Debug, Error, etc.) plus +// extended trace levels (Trace, Trace1-4) for fine-grained debugging. package logger import ( @@ -6,6 +35,9 @@ import ( "log/slog" ) +// Logger is the main interface for logging operations. +// It provides methods for different log levels and supports +// structured logging with key-value pairs. type Logger interface { Warnf(format string, args ...interface{}) Infof(format string, args ...interface{}) @@ -13,6 +45,12 @@ type Logger interface { Debugf(format string, args ...interface{}) Tracef(format string, args ...interface{}) Fatalf(format string, args ...interface{}) + // WithValues returns a new Logger with additional key-value pairs + // that will be included in all subsequent log messages. + // + // Example: + // log := logger.WithValues("component", "auth", "version", "1.0") + // log.Infof("User logged in") // Will include component=auth version=1.0 WithValues(keysAndValues ...interface{}) Logger IsTraceEnabled() bool IsDebugEnabled() bool @@ -20,14 +58,33 @@ type Logger interface { GetLevel() LogLevel SetLogLevel(level any) SetMinLogLevel(level any) + // V returns a Verbose logger that only logs if the specified level is enabled. + // Level can be an integer or a named level (Debug, Trace, etc.). + // + // Example: + // logger.V(2).Infof("This only logs at verbosity 2+") + // logger.V(logger.Trace).Infof("This only logs at trace level") V(level any) Verbose WithV(level any) Logger + // Named returns a new Logger with the specified name added to the logging context. + // This helps identify which component or subsystem generated the log. + // + // Example: + // dbLogger := logger.Named("database") + // dbLogger.Infof("Connected to database") // Logs with name="database" Named(name string) Logger WithoutName() Logger WithSkipReportLevel(i int) Logger GetSlogLogger() *slog.Logger } +// Verbose provides conditional logging based on verbosity levels. +// It's returned by Logger.V() and only logs if the specified level is enabled. +// +// Example: +// +// // Only logs if verbosity is 2 or higher +// logger.V(2).Infof("Detailed debug information: %v", data) type Verbose interface { io.Writer Infof(format string, args ...interface{}) @@ -35,6 +92,8 @@ type Verbose interface { Enabled() bool } +// LogLevel represents the severity of a log message. +// Higher values indicate more verbose logging. type LogLevel int const ( diff --git a/logger/logger_test.go b/logger/logger_test.go index 4d01fa5..e51cc20 100644 --- a/logger/logger_test.go +++ b/logger/logger_test.go @@ -11,6 +11,12 @@ import ( var log = StandardLogger() var _ = ginkgo.Describe("LogLevel Parsing", func() { + ginkgo.It("Logger Names", func() { + gomega.Expect(camelCaseWords("JohnDoe")).To((gomega.ContainElements("John", "Doe"))) + gomega.Expect(camelCaseWords("johnDoe")).To((gomega.ContainElements("john", "Doe"))) + gomega.Expect(camelCaseWords("john-doe")).To((gomega.ContainElements("john-doe"))) + + }) ginkgo.It("Default log level", func() { gomega.Expect(GetLogger().GetLevel()).To(gomega.Equal(Info)) }) diff --git a/logger/slog.go b/logger/slog.go index 5a63fbe..5f6ffce 100644 --- a/logger/slog.go +++ b/logger/slog.go @@ -162,6 +162,13 @@ func camelCaseWords(s string) []string { return lo.Map(strings.Fields(result.String()), func(s string, _ int) string { return strings.TrimSpace(s) }) } +// GetLogger returns a logger instance, optionally with the specified names. +// If no names are provided, returns the root logger. +// Multiple names create a hierarchical logger (e.g., GetLogger("app", "db") creates "app.db"). +// +// Example: +// dbLogger := logger.GetLogger("database") +// apiLogger := logger.GetLogger("api", "v1") func GetLogger(names ...string) *SlogLogger { parent, _ := namedLoggers.Load(rootName) if len(names) == 0 { diff --git a/properties/properties.go b/properties/properties.go index c94624e..217a00c 100644 --- a/properties/properties.go +++ b/properties/properties.go @@ -1,3 +1,54 @@ +// Package properties provides a thread-safe properties management system with +// file loading, hot-reloading, and change notification support. +// +// The package offers a simple key-value store for application configuration +// with support for loading from properties files, command-line overrides, +// and dynamic updates with file watching. +// +// Key features: +// - Load properties from files (key=value format) +// - Command-line property overrides via -P flag +// - File watching with automatic hot-reloading +// - Thread-safe operations with read-write locks +// - Change listeners for reactive configuration +// - Type-safe accessors for common data types +// +// Basic usage: +// +// // Load properties from file +// err := properties.LoadFile("app.properties") +// if err != nil { +// log.Fatal(err) +// } +// +// // Get property values with defaults +// host := properties.String("localhost", "server.host") +// port := properties.Int(8080, "server.port") +// debug := properties.On(false, "debug.enabled", "debug") +// timeout := properties.Duration(30*time.Second, "request.timeout") +// +// // Set properties dynamically +// properties.Set("api.key", "secret-key") +// +// // Register change listener +// properties.RegisterListener(func(p *properties.Properties) { +// log.Println("Properties updated") +// }) +// +// Command-line usage: +// +// ./app -P db.host=localhost -P db.port=5432 +// +// Properties file format: +// +// # Comments start with # +// server.host=localhost +// server.port=8080 +// debug.enabled=true +// request.timeout=30s +// +// The package maintains a global instance for convenience, but you can also +// create isolated Properties instances for different configuration contexts. package properties import ( @@ -18,25 +69,40 @@ import ( var commandlineProperties map[string]string +// Global is the default properties instance used by package-level functions. +// It's automatically initialized and ready to use. var Global = &Properties{ m: make(map[string]string), } +// LoadFile is a convenience function that loads properties from a file +// into the global properties instance. It's equivalent to Global.LoadFile(filename). var LoadFile = func(filename string) error { return Global.LoadFile(filename) } +// BindFlags binds the -P/--properties flag to the given flag set, allowing +// properties to be set via command line. +// +// Example: +// +// flags := pflag.NewFlagSet("app", pflag.ContinueOnError) +// properties.BindFlags(flags) +// flags.Parse(os.Args[1:]) +// // Now you can use: ./app -P key1=value1 -P key2=value2 func BindFlags(flags *pflag.FlagSet) { flags.StringToStringVarP(&commandlineProperties, "properties", "P", nil, "System properties") } +// Properties represents a thread-safe key-value store for application configuration. +// It supports loading from files, dynamic updates, file watching, and change notifications. type Properties struct { - m map[string]string - filename string - listeners []func(*Properties) - lock sync.RWMutex - close func() - Reload func() + m map[string]string // The property map + filename string // Currently loaded file + listeners []func(*Properties) // Change listeners + lock sync.RWMutex // Protects concurrent access + close func() // Cleanup function for file watcher + Reload func() // Function to manually trigger reload } func (p *Properties) RegisterListener(fn func(*Properties)) { @@ -60,6 +126,7 @@ func (p *Properties) GetAll() map[string]string { p.lock.RLock() defer p.lock.RUnlock() m := p.m + //command line properties take priority for k, v := range commandlineProperties { m[k] = v } @@ -67,6 +134,7 @@ func (p *Properties) GetAll() map[string]string { } func (p *Properties) Get(key string) string { + //command line properties take priority if v, ok := commandlineProperties[key]; ok { return v } diff --git a/text/bytes.go b/text/bytes.go index 642fdd2..6806eef 100644 --- a/text/bytes.go +++ b/text/bytes.go @@ -21,6 +21,50 @@ is subject to the terms and conditions of each subcomponent's license, as noted in the LICENSE file. */ +// Package text provides utilities for text formatting, humanization, and manipulation. +// +// The package offers functions to format bytes and numbers into human-readable strings, +// handle durations, safely read from io.Reader, and indent text output. It includes +// utilities commonly needed for CLI tools and logging. +// +// Key features: +// - Human-readable byte formatting (e.g., "10M", "12.5K") +// - Human-readable number formatting with metric suffixes +// - Duration parsing and formatting +// - Text indentation utilities +// - Safe reader utilities +// +// Byte formatting example: +// +// size := int64(1536) +// fmt.Println(text.HumanizeBytes(size)) // "1.5K" +// +// size = 1073741824 +// fmt.Println(text.HumanizeBytes(size)) // "1G" +// +// Number formatting example: +// +// count := 1500 +// fmt.Println(text.HumanizeInt(count)) // "1.5k" +// +// count = 2000000 +// fmt.Println(text.HumanizeInt(count)) // "2m" +// +// Duration example: +// +// d := 3*time.Hour + 30*time.Minute +// fmt.Println(text.HumanizeDuration(d)) // "3h30m" +// +// age := text.Age(time.Now().Add(-24 * time.Hour)) +// fmt.Println(age) // "1d" +// +// Indentation example: +// +// indented := text.String(" ", "line1\nline2\nline3") +// // Result: +// // line1 +// // line2 +// // line3 package text import ( @@ -40,18 +84,26 @@ const ( EXABYTE ) -// ByteSize returns a human-readable byte string of the form 10M, 12.5K, and so forth. The following units are available: +// HumanizeBytes returns a human-readable byte string of the form 10M, 12.5K, and so forth. +// The following units are available: // -// E: Exabyte -// P: Petabyte -// T: Terabyte -// G: Gigabyte -// M: Megabyte -// K: Kilobyte +// E: Exabyte (1024^6 bytes) +// P: Petabyte (1024^5 bytes) +// T: Terabyte (1024^4 bytes) +// G: Gigabyte (1024^3 bytes) +// M: Megabyte (1024^2 bytes) +// K: Kilobyte (1024 bytes) // B: Byte // // The unit that results in the smallest number greater than or equal to 1 is always chosen. -// Input is the size in bytes. +// The function accepts uint, uint64, int, int64, or string as input. +// +// Examples: +// +// HumanizeBytes(1024) // "1K" +// HumanizeBytes(1536) // "1.5K" +// HumanizeBytes(1048576) // "1M" +// HumanizeBytes("1073741824") // "1G" func HumanizeBytes(size interface{}) string { unit := "" var bytes uint64 @@ -111,6 +163,17 @@ const ( GIGA = 1000 * MEGA ) +// HumanizeInt formats integers with metric suffixes for readability. +// It uses decimal (base-10) units: k for thousands, m for millions, b for billions. +// +// The function accepts uint, uint64, int, int64, or string as input. +// +// Examples: +// +// HumanizeInt(1500) // "1.5k" +// HumanizeInt(2000000) // "2m" +// HumanizeInt(1500000000) // "1.5b" +// HumanizeInt("1000") // "1k" func HumanizeInt(size interface{}) string { unit := "" var val uint64 diff --git a/text/duration.go b/text/duration.go index f775c67..a9e8f22 100644 --- a/text/duration.go +++ b/text/duration.go @@ -6,21 +6,42 @@ import ( "github.com/flanksource/commons/duration" ) -// Returns a string representing of a duration in the form "3d1h3m". +// HumanizeDuration returns a string representing of a duration in the form "3d1h3m". // // Leading zero units are omitted. As a special case, durations less than one // second format use a smaller unit (milli-, micro-, or nanoseconds) to ensure // that the leading digit is non-zero. Duration more than a day or more than a // week lose granularity and are truncated to resp. days-hours-minutes and // weeks-days-hours. The zero duration formats as 0s. +// +// Examples: +// +// HumanizeDuration(3*time.Hour + 30*time.Minute) // "3h30m" +// HumanizeDuration(24*time.Hour) // "1d" +// HumanizeDuration(168*time.Hour + 12*time.Hour) // "1w12h" func HumanizeDuration(d time.Duration) string { return duration.Duration(d).String() } +// Age returns a human-readable string representing the time elapsed since the given time. +// It's a convenience function that combines time.Since with HumanizeDuration. +// +// Example: +// +// created := time.Now().Add(-24 * time.Hour) +// fmt.Printf("Created %s ago", text.Age(created)) // "Created 1d ago" func Age(d time.Time) string { return HumanizeDuration(time.Since(d)) } +// ParseDuration parses a duration string with support for extended units like days and weeks. +// Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h", "d", "w", "y". +// +// Examples: +// +// d, _ := ParseDuration("3d12h") // 3 days 12 hours +// d, _ := ParseDuration("1w") // 1 week +// d, _ := ParseDuration("2h30m") // 2 hours 30 minutes func ParseDuration(val string) (*time.Duration, error) { d, err := duration.ParseDuration(val) if err != nil { diff --git a/text/helper.go b/text/helper.go index f461ab1..7f903ac 100644 --- a/text/helper.go +++ b/text/helper.go @@ -4,6 +4,17 @@ import ( "io" ) +// SafeRead reads all data from an io.Reader and returns it as a string. +// If an error occurs during reading, it returns an empty string instead of +// propagating the error. This is useful when you want to ensure a read +// operation never fails, such as in logging or display contexts. +// +// Example: +// +// resp, _ := http.Get("https://example.com") +// defer resp.Body.Close() +// content := text.SafeRead(resp.Body) +// // content will be empty string if read fails func SafeRead(r io.Reader) string { data, _ := io.ReadAll(r) return string(data) diff --git a/utils/lock.go b/utils/lock.go index 198a30f..4382559 100644 --- a/utils/lock.go +++ b/utils/lock.go @@ -8,6 +8,8 @@ import ( "golang.org/x/sync/semaphore" ) +// Unlocker provides a method to release a lock. +// It follows the standard pattern for lock guards in Go. type Unlocker interface { Release() } @@ -20,10 +22,55 @@ func (u *unlocker) Release() { u.lock.Release(1) } +// NamedLock provides a mechanism for acquiring locks by name, allowing +// different parts of the code to synchronize on string identifiers. +// This is useful for resource-based locking where you want to ensure +// exclusive access to resources identified by strings (e.g., file paths, +// user IDs, resource names). +// +// The implementation uses semaphores internally, providing timeout support +// and preventing goroutine leaks. type NamedLock struct { locks sync.Map } +// TryLock attempts to acquire a lock with the given name within the specified timeout. +// If successful, it returns an Unlocker that must be used to release the lock. +// If the lock cannot be acquired within the timeout, it returns nil. +// +// The lock is exclusive - only one goroutine can hold a lock with a given name +// at any time. +// +// Example: +// +// lock := &NamedLock{} +// +// // Try to acquire lock with 5-second timeout +// if unlocker := lock.TryLock("user-123", 5*time.Second); unlocker != nil { +// defer unlocker.Release() +// // Critical section - exclusive access to user-123 +// updateUser("123") +// } else { +// // Could not acquire lock within timeout +// return errors.New("resource is locked") +// } +// +// Multiple locks example: +// +// // Different names don't block each other +// go func() { +// if u := lock.TryLock("resource-A", time.Second); u != nil { +// defer u.Release() +// // Work with resource A +// } +// }() +// +// go func() { +// if u := lock.TryLock("resource-B", time.Second); u != nil { +// defer u.Release() +// // Work with resource B +// } +// }() func (n *NamedLock) TryLock(name string, timeout time.Duration) Unlocker { o, _ := n.locks.LoadOrStore(name, semaphore.NewWeighted(1)) lock := o.(*semaphore.Weighted) diff --git a/utils/time.go b/utils/time.go index 6c589a3..7d11475 100644 --- a/utils/time.go +++ b/utils/time.go @@ -2,6 +2,31 @@ package utils import "time" +// ParseTime attempts to parse a time string using multiple common formats. +// It tries various formats in order and returns the first successful parse. +// Returns nil if no format matches. +// +// Supported formats: +// - RFC3339: "2006-01-02T15:04:05Z07:00" +// - RFC3339Nano: "2006-01-02T15:04:05.999999999Z07:00" +// - ANSIC: "Mon Jan _2 15:04:05 2006" +// - DateTime: "2006-01-02 15:04:05" +// - DateOnly: "2006-01-02" +// - ISO8601 without timezone: "2006-01-02T15:04:05" +// - MySQL datetime: "2006-01-02 15:04:05" +// +// Example: +// +// t1 := utils.ParseTime("2023-12-25T10:30:00Z") // RFC3339 +// t2 := utils.ParseTime("2023-12-25 10:30:00") // MySQL format +// t3 := utils.ParseTime("2023-12-25") // Date only +// t4 := utils.ParseTime("Mon Jan 2 15:04:05 2006") // ANSIC +// +// if t := utils.ParseTime(userInput); t != nil { +// fmt.Printf("Parsed time: %v\n", t) +// } else { +// fmt.Println("Invalid time format") +// } func ParseTime(t string) *time.Time { formats := []string{ time.RFC3339, diff --git a/utils/utils.go b/utils/utils.go index 79ad389..c58f9cc 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,3 +1,62 @@ +// Package utils provides common utility functions for Go applications. +// +// The package includes helpers for pointer operations, string manipulation, +// randomization, template interpolation, time parsing, and named locks. These +// utilities address common patterns in Go programming and reduce boilerplate code. +// +// Key features: +// - Generic pointer and dereference utilities +// - Coalesce function for finding first non-zero value +// - Environment variable helpers +// - Random string and key generation +// - Template string interpolation +// - Flexible time parsing with multiple formats +// - Named lock implementation for synchronization +// +// Pointer utilities: +// +// // Create pointer to value +// strPtr := utils.Ptr("hello") +// intPtr := utils.Ptr(42) +// +// // Safely dereference (returns zero value if nil) +// val := utils.Deref(strPtr) // "hello" +// val2 := utils.Deref(nil) // "" (zero value) +// +// Coalesce example: +// +// // Returns first non-zero value +// result := utils.Coalesce("", "", "value", "ignored") // "value" +// port := utils.Coalesce(0, 0, 8080, 9090) // 8080 +// +// Random generation: +// +// // Generate random hex key +// apiKey := utils.RandomKey(32) +// +// // Generate random alphanumeric string +// sessionID := utils.RandomString(16) +// +// Template interpolation: +// +// vars := map[string]string{"name": "World", "time": "today"} +// result := utils.Interpolate("Hello {{.name}}, how are you {{.time}}?", vars) +// // "Hello World, how are you today?" +// +// Time parsing: +// +// // Parse time with automatic format detection +// t := utils.ParseTime("2023-12-25T10:30:00Z") // RFC3339 +// t2 := utils.ParseTime("2023-12-25 10:30:00") // MySQL format +// t3 := utils.ParseTime("2023-12-25") // Date only +// +// Named locks: +// +// lock := &utils.NamedLock{} +// if unlocker := lock.TryLock("resource-1", 5*time.Second); unlocker != nil { +// defer unlocker.Release() +// // Critical section +// } package utils import ( @@ -15,10 +74,32 @@ import ( log "github.com/sirupsen/logrus" ) +// Ptr returns a pointer to the given value. +// This is a generic helper function useful for creating pointers to literals +// or values that need to be passed as pointers to functions. +// +// Example: +// +// // Instead of: +// temp := "hello" +// ptr := &temp +// +// // You can write: +// ptr := utils.Ptr("hello") func Ptr[T any](value T) *T { return &value } +// Deref safely dereferences a pointer, returning the zero value if the pointer is nil. +// This prevents nil pointer panics and simplifies nil checking. +// +// Example: +// +// var strPtr *string = nil +// val := utils.Deref(strPtr) // Returns "" (zero value for string) +// +// strPtr = utils.Ptr("hello") +// val = utils.Deref(strPtr) // Returns "hello" func Deref[T any](v *T) T { if v == nil { var zero T @@ -28,7 +109,20 @@ func Deref[T any](v *T) T { return *v } -// Coalesce returns the first non-zero element +// Coalesce returns the first non-zero value from the provided arguments. +// This is similar to the COALESCE function in SQL and the nullish coalescing +// operator (??) in other languages. +// +// Example: +// +// // String coalescing +// name := utils.Coalesce("", "", "John", "Jane") // Returns "John" +// +// // Number coalescing +// port := utils.Coalesce(0, 0, 8080, 9090) // Returns 8080 +// +// // With variables +// result := utils.Coalesce(config.URL, os.Getenv("API_URL"), "http://localhost") func Coalesce[T comparable](arr ...T) T { var zeroVal T for _, item := range arr { @@ -40,7 +134,20 @@ func Coalesce[T comparable](arr ...T) T { return zeroVal } -// GetEnvOrDefault returns the first non-empty environment variable +// GetEnvOrDefault returns the value of the first non-empty environment variable +// from the provided list of names. This is useful for checking multiple possible +// environment variable names or providing fallback options. +// +// Example: +// +// // Check multiple possible names +// dbHost := utils.GetEnvOrDefault("DATABASE_HOST", "DB_HOST", "POSTGRES_HOST") +// +// // With fallback handling +// apiKey := utils.GetEnvOrDefault("API_KEY", "SECRET_KEY") +// if apiKey == "" { +// apiKey = "default-key" +// } func GetEnvOrDefault(names ...string) string { for _, name := range names { if val := os.Getenv(name); val != "" {