feat(bundler): support gitignore-style negation in .juliabundleignore

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 * The `expose` argument of `submit_job` now also accepts a `JuliaHub.JobRemoteAccess` object, which additionally allows configuring who can access the exposed port (`JuliaHub.JobAccessMode`) and a fixed DNS prefix for the job. Ports can now also be exposed on package application jobs. ([#154], [#158])
 * `JuliaHub.PackageJob` now supports overriding the job image (`image`) and selecting which revision of the application to launch (`revision`, see `JuliaHub.PackageAppRevision`). ([#154])
+* `.juliabundleignore` now supports `.gitignore`-style negation: a pattern starting with `!` re-includes paths excluded by an earlier pattern, with last-matching-pattern-wins semantics. Comment (`#`) and blank lines are also now skipped. ([#130])
 
 ### Fixed
 
@@ -242,6 +243,7 @@ Initial package release.
 [#117]: https://github.com/JuliaComputing/JuliaHub.jl/issues/117
 [#124]: https://github.com/JuliaComputing/JuliaHub.jl/issues/124
 [#127]: https://github.com/JuliaComputing/JuliaHub.jl/issues/127
+[#130]: https://github.com/JuliaComputing/JuliaHub.jl/issues/130
 [#132]: https://github.com/JuliaComputing/JuliaHub.jl/issues/132
 [#133]: https://github.com/JuliaComputing/JuliaHub.jl/issues/133
 [#149]: https://github.com/JuliaComputing/JuliaHub.jl/issues/149

docs/src/reference/job-submission.md

@@ -150,6 +150,33 @@ This `.juliabundleignore` will ignore:
 
 You can also have additional `.juliabundleignore` files in subdirectories and they will only apply to those directories and their subdirectories.
 
+#### Negating patterns (`!`)
+
+Like `.gitignore`, a pattern starting with `!` re-includes files that an earlier
+pattern excluded. Patterns are applied top to bottom and the **last matching
+pattern wins**. This lets you ignore everything and then add back only what you
+need:
+
+```
+*
+!programs/
+!programs/*
+!Project.toml
+!Manifest.toml
+```
+
+This bundles only the `programs/` directory (and everything under it) plus the
+top-level `Project.toml` and `Manifest.toml`, and ignores everything else.
+
+!!! note "Re-include the parent directory first"
+    You cannot re-include a file whose parent directory is still excluded — the
+    bundler skips excluded directories entirely and never descends into them (the
+    same limitation `git` has). Re-include the directory itself (e.g. `!programs/`)
+    before, or together with, its contents (`!programs/*`).
+
+Lines beginning with `#` are treated as comments, and blank lines are ignored.
+Use `\!` or `\#` to match a file whose name literally begins with `!` or `#`.
+
 ### Specifying the job image
 
 JuliaHub batch jobs can run in various container images.

src/PackageBundler/utils.jl

@@ -73,17 +73,35 @@ function is_subpath(dir, subpath)
     return startswith(norm_subpath, norm_dir)
 end
 
+# Parse one line of a `.juliabundleignore` file into an ordered rule, mirroring
+# `.gitignore` syntax. Returns `nothing` for blank lines and `#` comments. A
+# leading `!` marks a negated (re-include) rule; a leading `\#` or `\!` escapes a
+# literal `#`/`!`. The remainder is compiled with `Glob.FilenameMatch`, so glob
+# and trailing-slash directory patterns behave exactly as before.
+function _parse_bundleignore_line(line)
+    s = strip(line)
+    (isempty(s) || startswith(s, '#')) && return nothing
+    negated = false
+    if startswith(s, '!')
+        negated = true
+        s = chop(s; head=1, tail=0)
+    elseif startswith(s, "\\!") || startswith(s, "\\#")
+        s = chop(s; head=1, tail=0)
+    end
+    return (; negated, pattern=Glob.FilenameMatch(String(s)))
+end
+
 function get_bundleignore(file, top)
     dir = dirname(file)
-    patterns = Set{Any}()
+    rules = @NamedTuple{negated::Bool, pattern::Glob.FilenameMatch}[]
     try
         while true
             if isfile(joinpath(dir, ".juliabundleignore"))
-                union!(
-                    patterns,
-                    Glob.FilenameMatch.(strip.(readlines(joinpath(dir, ".juliabundleignore")))),
-                )
-                return patterns, dir
+                for line in readlines(joinpath(dir, ".juliabundleignore"))
+                    rule = _parse_bundleignore_line(line)
+                    rule !== nothing && push!(rules, rule)
+                end
+                return rules, dir
             end
             if dir == dirname(dir) || dir == top
                 break
@@ -93,7 +111,7 @@ function get_bundleignore(file, top)
     catch err
         @warn "Internal error" exception = (err, catch_backtrace())
     end
-    return patterns, top
+    return rules, top
 end
 
 """
@@ -139,18 +157,26 @@ function path_filterer(top)
             return false
         end
 
-        patterns, ignorepath = get_bundleignore(path, top)
-
-        rpath = relpath(path, ignorepath)
-
-        return !(
-            any(p -> occursin(p, sanitize_windows_path(rpath)), patterns) ||
-            # directories specifically can be excluded by patterns that end with a
-            # path separator, and to match them in case `path` does not have that
-            # path separator appended, we append it ourselves before matching
-            isdir(path) &&
-            any(p -> occursin(p, sanitize_windows_path(joinpath(rpath, ""))), patterns)
-        )
+        rules, ignorepath = get_bundleignore(path, top)
+
+        rel = relpath(path, ignorepath)
+        rpath = sanitize_windows_path(rel)
+        # directories can be matched by patterns ending in a path separator, so we
+        # also test the path with a trailing separator appended.
+        rpath_dir = sanitize_windows_path(joinpath(rel, ""))
+        isdirp = isdir(path)
+
+        # gitignore semantics: evaluate rules in file order; the last rule that
+        # matches decides. A plain (non-negated) rule that matches excludes the
+        # path; a negated (`!`) rule that matches re-includes it. Paths matched by
+        # no rule are kept.
+        included = true
+        for rule in rules
+            if occursin(rule.pattern, rpath) || (isdirp && occursin(rule.pattern, rpath_dir))
+                included = rule.negated
+            end
+        end
+        return included
     end
 end
 

test/packagebundler.jl

@@ -267,6 +267,104 @@ end
     end
 end
 
+@testset "path_filterer negation" begin
+    # gitignore-style: ignore everything, then re-include specific paths.
+    root = mktempdir()
+    write(
+        joinpath(root, ".juliabundleignore"),
+        """
+*
+!programs/
+!programs/*
+!Project.toml
+!Manifest.toml
+""",
+    )
+    mkpath(joinpath(root, "programs", "appA"))
+    touch(joinpath(root, "programs", "appA", "main.jl"))
+    touch(joinpath(root, "programs", "loose.txt"))
+    mkpath(joinpath(root, "secret"))
+    touch(joinpath(root, "secret", "data.csv"))
+    touch(joinpath(root, "Project.toml"))
+    touch(joinpath(root, "Manifest.toml"))
+    touch(joinpath(root, "README.md"))
+    pred = JuliaHub._PackageBundler.path_filterer(root)
+
+    # Re-included: parent dir (so Tar descends), its subdir, deep file, loose file.
+    @test pred(joinpath(root, "programs"))
+    @test pred(joinpath(root, "programs", "appA"))
+    @test pred(joinpath(root, "programs", "appA", "main.jl"))
+    @test pred(joinpath(root, "programs", "loose.txt"))
+    @test pred(joinpath(root, "Project.toml"))
+    @test pred(joinpath(root, "Manifest.toml"))
+    # Still excluded: unlisted dir and file.
+    @test !pred(joinpath(root, "secret"))
+    @test !pred(joinpath(root, "README.md"))
+
+    # Last-matching rule wins (order matters).
+    root2 = mktempdir()
+    write(
+        joinpath(root2, ".juliabundleignore"),
+        """
+*.log
+!keep*.log
+keep-tmp.log
+""",
+    )
+    touch(joinpath(root2, "app.log"))
+    touch(joinpath(root2, "keep-final.log"))
+    touch(joinpath(root2, "keep-tmp.log"))
+    pred2 = JuliaHub._PackageBundler.path_filterer(root2)
+    @test !pred2(joinpath(root2, "app.log"))       # ignored by *.log
+    @test pred2(joinpath(root2, "keep-final.log")) # re-included by !keep*.log
+    @test !pred2(joinpath(root2, "keep-tmp.log"))  # re-ignored by later keep-tmp.log
+
+    # Comments and blank lines are inert.
+    root3 = mktempdir()
+    write(joinpath(root3, ".juliabundleignore"), "# just a comment\n\n*.tmp\n")
+    touch(joinpath(root3, "a.tmp"))
+    touch(joinpath(root3, "b.jl"))
+    pred3 = JuliaHub._PackageBundler.path_filterer(root3)
+    @test !pred3(joinpath(root3, "a.tmp"))
+    @test pred3(joinpath(root3, "b.jl"))
+end
+
+@testset "bundle negation (Tar)" begin
+    root = mktempdir()
+    write(
+        joinpath(root, ".juliabundleignore"),
+        """
+*
+!programs/
+!programs/*
+!Project.toml
+!Manifest.toml
+""",
+    )
+    mkpath(joinpath(root, "programs", "appA"))
+    write(joinpath(root, "programs", "appA", "main.jl"), "println(1)")
+    write(joinpath(root, "programs", "notes.txt"), "hi")
+    mkpath(joinpath(root, "secret"))
+    write(joinpath(root, "secret", "data.csv"), "x,y")
+    write(joinpath(root, "Project.toml"), "name = \"X\"\n")
+    write(joinpath(root, "Manifest.toml"), "")
+    write(joinpath(root, "README.md"), "readme")
+
+    tb = tempname()
+    Tar.create(JuliaHub._PackageBundler.path_filterer(root), root, tb)
+    ex = mktempdir()
+    Tar.extract(tb, ex)
+
+    # Non-empty and contains exactly the re-included content.
+    @test isfile(joinpath(ex, "programs", "appA", "main.jl"))
+    @test isfile(joinpath(ex, "programs", "notes.txt"))
+    @test isfile(joinpath(ex, "Project.toml"))
+    @test isfile(joinpath(ex, "Manifest.toml"))
+    # Excluded paths must not be present.
+    @test !ispath(joinpath(ex, "secret"))
+    @test !isfile(joinpath(ex, "README.md"))
+end
+
 @testset "cp_skip_dangling_symlinks" begin
     mktempdir() do src
         # Regular file