diff --git a/git-sensor/env_gen.json b/git-sensor/env_gen.json index ee9920fa8..b82029354 100644 --- a/git-sensor/env_gen.json +++ b/git-sensor/env_gen.json @@ -1 +1 @@ -[{"Category":"DEVTRON","Fields":[{"Env":"ANALYTICS_DEBUG","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP","EnvType":"string","EnvValue":"git-sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"900","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"COMMIT_STATS_TIMEOUT_IN_SEC","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_FILE_STATS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_HISTORY_COUNT","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GOGIT_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIN_LIMIT_FOR_PVC","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"git_sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_DURATION","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_WORKER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_GRPC_PORT","EnvType":"int","EnvValue":"8081","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_REST_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI_ANALYTICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"DEVTRON","Fields":[{"Env":"ANALYTICS_DEBUG","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP","EnvType":"string","EnvValue":"git-sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"900","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"COMMIT_STATS_TIMEOUT_IN_SEC","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_FILE_STATS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_MANUAL_GIT_GC","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_HISTORY_COUNT","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GOGIT_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MANUAL_GIT_GC_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIN_LIMIT_FOR_PVC","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"git_sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_DURATION","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_WORKER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_GRPC_PORT","EnvType":"int","EnvValue":"8081","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_REST_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI_ANALYTICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/git-sensor/env_gen.md b/git-sensor/env_gen.md index f71486a1e..a7eeb8b86 100644 --- a/git-sensor/env_gen.md +++ b/git-sensor/env_gen.md @@ -11,10 +11,12 @@ | CONSUMER_CONFIG_JSON | string | | | | false | | DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false | | ENABLE_FILE_STATS | bool |false | | | false | + | ENABLE_MANUAL_GIT_GC | bool |false | | | false | | ENABLE_STATSVIZ | bool |false | | | false | | GIT_HISTORY_COUNT | int |15 | | | false | | GOGIT_TIMEOUT_SECONDS | int |10 | | | false | | LOG_LEVEL | int |0 | | | false | + | MANUAL_GIT_GC_TIMEOUT_SECONDS | int |300 | | | false | | MIN_LIMIT_FOR_PVC | int |1 | | | false | | NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false | | NATS_MSG_BUFFER_SIZE | int |-1 | | | false | diff --git a/git-sensor/internals/Configuration.go b/git-sensor/internals/Configuration.go index ba4e2839b..abc132530 100644 --- a/git-sensor/internals/Configuration.go +++ b/git-sensor/internals/Configuration.go @@ -29,6 +29,8 @@ type Configuration struct { CliCmdTimeoutGlobal int `env:"CLI_CMD_TIMEOUT_GLOBAL_SECONDS" envDefault:"900"` CliCmdTimeoutJson string `env:"CLI_CMD_TIMEOUT_JSON" envDefault:""` GoGitTimeout int `env:"GOGIT_TIMEOUT_SECONDS" envDefault:"10" ` + EnableManualGitGc bool `env:"ENABLE_MANUAL_GIT_GC" envDefault:"false"` + ManualGitGcTimeout int `env:"MANUAL_GIT_GC_TIMEOUT_SECONDS" envDefault:"300"` } func ParseConfiguration() (*Configuration, error) { diff --git a/git-sensor/pkg/git/GitBaseManager.go b/git-sensor/pkg/git/GitBaseManager.go index 6802b247e..c1f8145d1 100644 --- a/git-sensor/pkg/git/GitBaseManager.go +++ b/git-sensor/pkg/git/GitBaseManager.go @@ -30,6 +30,7 @@ import ( "os" "os/exec" "regexp" + "slices" "strings" "time" ) @@ -406,6 +407,13 @@ func (impl *GitManagerBaseImpl) createCmdWithContext(ctx GitContext, name string } else { impl.logger.Infow("sucess cancel command", "name", name, "arg", arg) } + + // Run git gc --prune=now to clean up temp.pack files left by killed git commands + if impl.conf.EnableManualGitGc && slices.Contains(arg, "fetch") { + if rootDir := impl.extractRootDirFromArgs(arg); rootDir != "" { + impl.runGitGcWithTimeout(rootDir, time.Duration(impl.conf.ManualGitGcTimeout)*time.Second) + } + } return err } return cmd, newCtx, cancel @@ -431,3 +439,35 @@ func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name output, errMsg, err := impl.runCommandWithCred(cmd, newGitCtx, gitContext.Username, gitContext.Password, tlsPathInfo) return output, errMsg, err } + +// extractRootDirFromArgs extracts the root directory from git command arguments. +// It looks for the -C flag which specifies the directory to run git commands in. +func (impl *GitManagerBaseImpl) extractRootDirFromArgs(args []string) string { + for i, arg := range args { + if arg == "-C" && i+1 < len(args) { + return args[i+1] + } + } + return "" +} + +// runGitGcWithTimeout runs git gc --prune=now with a timeout to clean up +// temporary pack files left behind when git commands are killed/cancelled. +func (impl *GitManagerBaseImpl) runGitGcWithTimeout(rootDir string, timeout time.Duration) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + impl.logger.Infow("running git gc --prune=now", "rootDir", rootDir, "timeout", timeout) + cmd := exec.CommandContext(ctx, "git", "-C", rootDir, "gc", "--prune=now") + cmd.Env = append(os.Environ(), "HOME=/dev/null") + output, err := cmd.CombinedOutput() + if err != nil { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + impl.logger.Warnw("git gc timed out", "rootDir", rootDir, "timeout", timeout) + } else { + impl.logger.Errorw("error running git gc", "rootDir", rootDir, "err", err, "output", string(output)) + } + } else { + impl.logger.Infow("git gc completed successfully", "rootDir", rootDir) + } +} diff --git a/git-sensor/wire_gen.go b/git-sensor/wire_gen.go index 3f2d145cb..a9a99e1f0 100644 --- a/git-sensor/wire_gen.go +++ b/git-sensor/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run -mod=mod github.com/google/wire/cmd/wire +//go:generate go run github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject