Skip to content

Commit 07252d4

Browse files
Shivam-nagar23Shivam-nagar23
authored
Merge pull request #362 from devtron-labs/feat-manual-gc
fix: added flag basis manual gc
2 parents d6692a4 + 03223d4 commit 07252d4

5 files changed

Lines changed: 46 additions & 2 deletions

File tree

‎git-sensor/env_gen.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
[{"Category":"DEVTRON","Fields":[{"Env":"ANALYTICS_DEBUG","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP","EnvType":"string","EnvValue":"git-sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"900","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"COMMIT_STATS_TIMEOUT_IN_SEC","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_FILE_STATS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_HISTORY_COUNT","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GOGIT_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIN_LIMIT_FOR_PVC","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"git_sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_DURATION","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_WORKER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_GRPC_PORT","EnvType":"int","EnvValue":"8081","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_REST_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI_ANALYTICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]}]
1+
[{"Category":"DEVTRON","Fields":[{"Env":"ANALYTICS_DEBUG","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"APP","EnvType":"string","EnvValue":"git-sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"900","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"COMMIT_STATS_TIMEOUT_IN_SEC","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_FILE_STATS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_MANUAL_GIT_GC","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ENABLE_STATSVIZ","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_HISTORY_COUNT","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GOGIT_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MANUAL_GIT_GC_TIMEOUT_SECONDS","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MIN_LIMIT_FOR_PVC","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"nats://devtron-nats.devtroncd:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"git_sensor","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_DURATION","EnvType":"int","EnvValue":"2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"POLL_WORKER","EnvType":"int","EnvValue":"5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_GRPC_PORT","EnvType":"int","EnvValue":"8081","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SERVER_REST_PORT","EnvType":"int","EnvValue":"8080","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI_ANALYTICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"}]}]

‎git-sensor/env_gen.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,12 @@
1111
| CONSUMER_CONFIG_JSON | string | | | | false |
1212
| DEFAULT_LOG_TIME_LIMIT | int64 |1 | | | false |
1313
| ENABLE_FILE_STATS | bool |false | | | false |
14+
| ENABLE_MANUAL_GIT_GC | bool |false | | | false |
1415
| ENABLE_STATSVIZ | bool |false | | | false |
1516
| GIT_HISTORY_COUNT | int |15 | | | false |
1617
| GOGIT_TIMEOUT_SECONDS | int |10 | | | false |
1718
| LOG_LEVEL | int |0 | | | false |
19+
| MANUAL_GIT_GC_TIMEOUT_SECONDS | int |300 | | | false |
1820
| MIN_LIMIT_FOR_PVC | int |1 | | | false |
1921
| NATS_MSG_ACK_WAIT_IN_SECS | int |120 | | | false |
2022
| NATS_MSG_BUFFER_SIZE | int |-1 | | | false |

‎git-sensor/internals/Configuration.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ type Configuration struct {
2929
CliCmdTimeoutGlobal int `env:"CLI_CMD_TIMEOUT_GLOBAL_SECONDS" envDefault:"900"`
3030
CliCmdTimeoutJson string `env:"CLI_CMD_TIMEOUT_JSON" envDefault:""`
3131
GoGitTimeout int `env:"GOGIT_TIMEOUT_SECONDS" envDefault:"10" `
32+
EnableManualGitGc bool `env:"ENABLE_MANUAL_GIT_GC" envDefault:"false"`
33+
ManualGitGcTimeout int `env:"MANUAL_GIT_GC_TIMEOUT_SECONDS" envDefault:"300"`
3234
}
3335

3436
func ParseConfiguration() (*Configuration, error) {

‎git-sensor/pkg/git/GitBaseManager.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"os"
3131
"os/exec"
3232
"regexp"
33+
"slices"
3334
"strings"
3435
"time"
3536
)
@@ -406,6 +407,13 @@ func (impl *GitManagerBaseImpl) createCmdWithContext(ctx GitContext, name string
406407
} else {
407408
impl.logger.Infow("sucess cancel command", "name", name, "arg", arg)
408409
}
410+
411+
// Run git gc --prune=now to clean up temp.pack files left by killed git commands
412+
if impl.conf.EnableManualGitGc && slices.Contains(arg, "fetch") {
413+
if rootDir := impl.extractRootDirFromArgs(arg); rootDir != "" {
414+
impl.runGitGcWithTimeout(rootDir, time.Duration(impl.conf.ManualGitGcTimeout)*time.Second)
415+
}
416+
}
409417
return err
410418
}
411419
return cmd, newCtx, cancel
@@ -431,3 +439,35 @@ func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name
431439
output, errMsg, err := impl.runCommandWithCred(cmd, newGitCtx, gitContext.Username, gitContext.Password, tlsPathInfo)
432440
return output, errMsg, err
433441
}
442+
443+
// extractRootDirFromArgs extracts the root directory from git command arguments.
444+
// It looks for the -C flag which specifies the directory to run git commands in.
445+
func (impl *GitManagerBaseImpl) extractRootDirFromArgs(args []string) string {
446+
for i, arg := range args {
447+
if arg == "-C" && i+1 < len(args) {
448+
return args[i+1]
449+
}
450+
}
451+
return ""
452+
}
453+
454+
// runGitGcWithTimeout runs git gc --prune=now with a timeout to clean up
455+
// temporary pack files left behind when git commands are killed/cancelled.
456+
func (impl *GitManagerBaseImpl) runGitGcWithTimeout(rootDir string, timeout time.Duration) {
457+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
458+
defer cancel()
459+
460+
impl.logger.Infow("running git gc --prune=now", "rootDir", rootDir, "timeout", timeout)
461+
cmd := exec.CommandContext(ctx, "git", "-C", rootDir, "gc", "--prune=now")
462+
cmd.Env = append(os.Environ(), "HOME=/dev/null")
463+
output, err := cmd.CombinedOutput()
464+
if err != nil {
465+
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
466+
impl.logger.Warnw("git gc timed out", "rootDir", rootDir, "timeout", timeout)
467+
} else {
468+
impl.logger.Errorw("error running git gc", "rootDir", rootDir, "err", err, "output", string(output))
469+
}
470+
} else {
471+
impl.logger.Infow("git gc completed successfully", "rootDir", rootDir)
472+
}
473+
}

‎git-sensor/wire_gen.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)