Merge pull request #982 from kane8n/add-filter-flag

Add "--filter" flag to enable git partial clone

Author: k8s-ci-robot

README.md

@@ -145,6 +145,7 @@ More documentation on specific topics can be [found here](./docs).
 ## Manual
 
 ```
+
 GIT-SYNC
 
 NAME
@@ -296,6 +297,14 @@ OPTIONS
             The timeout for the --exechook-command.  If not specifid, this
             defaults to 30 seconds ("30s").
 
+    --filter <string>, $GITSYNC_FILTER
+            Use partial clone with the specified filter.  This can reduce
+            the amount of data transferred when cloning large repositories.
+            Common values are 'blob:none' (omit all blobs, fetch on demand)
+            and 'tree:0' (omit all trees and blobs).  This is most effective
+            when combined with --depth and --sparse-checkout-file.  See
+            https://git-scm.com/docs/partial-clone for more information.
+
     --git <string>, $GITSYNC_GIT
             The git command to run (subject to PATH search, mostly for
             testing).  This defaults to "git".
@@ -425,7 +434,10 @@ OPTIONS
     --password-file <string>, $GITSYNC_PASSWORD_FILE
             The file from which the password or personal access token (see
             github docs) to use for git authentication (see --username) will be
-            read.  See also $GITSYNC_PASSWORD.
+            read.  The file is re-read before each sync attempt, allowing
+            git-sync to pick up token rotations automatically (e.g. when using
+            dynamic credentials from an external secrets system).
+            See also $GITSYNC_PASSWORD.
 
     --period <duration>, $GITSYNC_PERIOD
             How long to wait between sync attempts.  This must be at least

main.go

@@ -120,6 +120,7 @@ type repoSync struct {
 	repo           string         // remote repo to sync
 	ref            string         // the ref to sync
 	depth          int            // for shallow sync
+	filter         string         // for partial clone
 	submodules     submodulesMode // how to handle submodules
 	gc             gcMode         // garbage collection
 	link           absPath        // absolute path to the symlink to publish
@@ -167,6 +168,9 @@ func main() {
 	flDepth := pflag.Int("depth",
 		envInt(1, "GITSYNC_DEPTH", "GIT_SYNC_DEPTH"),
 		"create a shallow clone with history truncated to the specified number of commits")
+	flFilter := pflag.String("filter",
+		envString("", "GITSYNC_FILTER"),
+		"use partial clone with the specified filter (e.g. 'blob:none', 'tree:0')")
 	flSubmodules := pflag.String("submodules",
 		envString("recursive", "GITSYNC_SUBMODULES", "GIT_SYNC_SUBMODULES"),
 		"git submodule behavior: one of 'recursive', 'shallow', or 'off'")
@@ -725,6 +729,7 @@ func main() {
 		repo:         *flRepo,
 		ref:          *flRef,
 		depth:        *flDepth,
+		filter:       *flFilter,
 		submodules:   submodulesMode(*flSubmodules),
 		gc:           gcMode(*flGitGC),
 		link:         absLink,
@@ -1893,6 +1898,9 @@ func (git *repoSync) fetch(ctx context.Context, ref string) error {
 			args = append(args, "--unshallow")
 		}
 	}
+	if git.filter != "" {
+		args = append(args, "--filter", git.filter)
+	}
 	if _, _, err := git.Run(ctx, git.root, args...); err != nil {
 		return err
 	}
@@ -2520,6 +2528,14 @@ OPTIONS
             The timeout for the --exechook-command.  If not specifid, this
             defaults to 30 seconds ("30s").
 
+    --filter <string>, $GITSYNC_FILTER
+            Use partial clone with the specified filter.  This can reduce
+            the amount of data transferred when cloning large repositories.
+            Common values are 'blob:none' (omit all blobs, fetch on demand)
+            and 'tree:0' (omit all trees and blobs).  This is most effective
+            when combined with --depth and --sparse-checkout-file.  See
+            https://git-scm.com/docs/partial-clone for more information.
+
     --git <string>, $GITSYNC_GIT
             The git command to run (subject to PATH search, mostly for
             testing).  This defaults to "git".

test_e2e.sh

@@ -3345,6 +3345,93 @@ function e2e::sparse_checkout() {
     assert_file_eq "$ROOT/link/file2" "${FUNCNAME[0]}"
 }
 
+##############################################
+# Test filter (partial clone) with blob:none
+##############################################
+function e2e::filter_partial_clone_blob_none() {
+    GIT_SYNC \
+        --one-time \
+        --repo="file://$REPO" \
+        --root="$ROOT" \
+        --link="link" \
+        --filter="blob:none" \
+        --depth=0
+    assert_link_exists "$ROOT/link"
+    assert_file_exists "$ROOT/link/file"
+    assert_file_eq "$ROOT/link/file" "${FUNCNAME[0]}"
+
+    # Verify partial clone is in effect: git fetch --filter creates a
+    # .promisor marker alongside each pack file containing filtered objects.
+    local pack_dir
+    pack_dir="$(git -C "$ROOT/link" rev-parse --git-common-dir)/objects/pack"
+    if ! ls "$pack_dir"/*.promisor >/dev/null 2>&1; then
+        fail "expected .promisor pack marker (partial clone)"
+    fi
+}
+
+##############################################
+# Test filter with sparse-checkout
+##############################################
+function e2e::filter_with_sparse_checkout() {
+    echo "!/*" > "$WORK/sparseconfig"
+    echo "!/*/" >> "$WORK/sparseconfig"
+    echo "file2" >> "$WORK/sparseconfig"
+    echo "${FUNCNAME[0]}" > "$REPO/file2"
+    mkdir -p "$REPO/dir"
+    echo "${FUNCNAME[0]}" > "$REPO/dir/file3"
+    git -C "$REPO" add file2
+    git -C "$REPO" add dir
+    git -C "$REPO" commit -qam "${FUNCNAME[0]}"
+
+    GIT_SYNC \
+        --one-time \
+        --repo="file://$REPO" \
+        --root="$ROOT" \
+        --link="link" \
+        --filter="blob:none" \
+        --depth=1 \
+        --sparse-checkout-file="$WORK/sparseconfig"
+    assert_link_exists "$ROOT/link"
+    assert_file_exists "$ROOT/link/file2"
+    assert_file_absent "$ROOT/link/dir/file3"
+    assert_file_absent "$ROOT/link/dir"
+    assert_file_eq "$ROOT/link/file2" "${FUNCNAME[0]}"
+}
+
+##############################################
+# Test filter syncing across updates
+##############################################
+function e2e::filter_across_updates() {
+    # First sync
+    echo "${FUNCNAME[0]} 1" > "$REPO/file"
+    git -C "$REPO" commit -qam "${FUNCNAME[0]} 1"
+
+    GIT_SYNC \
+        --period=100ms \
+        --repo="file://$REPO" \
+        --filter="blob:none" \
+        --depth=0 \
+        --root="$ROOT" \
+        --link="link" \
+        &
+    wait_for_sync "${MAXWAIT}"
+    assert_link_exists "$ROOT/link"
+    assert_file_exists "$ROOT/link/file"
+    assert_file_eq "$ROOT/link/file" "${FUNCNAME[0]} 1"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 1
+    assert_metric_eq "${METRIC_FETCH_COUNT}" 1
+
+    # Move forward
+    echo "${FUNCNAME[0]} 2" > "$REPO/file"
+    git -C "$REPO" commit -qam "${FUNCNAME[0]} 2"
+    wait_for_sync "${MAXWAIT}"
+    assert_link_exists "$ROOT/link"
+    assert_file_exists "$ROOT/link/file"
+    assert_file_eq "$ROOT/link/file" "${FUNCNAME[0]} 2"
+    assert_metric_eq "${METRIC_GOOD_SYNC_COUNT}" 2
+    assert_metric_eq "${METRIC_FETCH_COUNT}" 2
+}
+
 ##############################################
 # Test additional git configs
 ##############################################