diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d02526504..c3aefd9c7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), * The `expose` argument of `submit_job` now also accepts a `JuliaHub.JobRemoteAccess` object, which additionally allows configuring who can access the exposed port (`JuliaHub.JobAccessMode`) and a fixed DNS prefix for the job. Ports can now also be exposed on package application jobs. ([#154], [#158]) * `JuliaHub.PackageJob` now supports overriding the job image (`image`) and selecting which revision of the application to launch (`revision`, see `JuliaHub.PackageAppRevision`). ([#154]) +* `.juliabundleignore` now supports `.gitignore`-style negation: a pattern starting with `!` re-includes paths excluded by an earlier pattern, with last-matching-pattern-wins semantics. Comment (`#`) and blank lines are also now skipped. ([#130]) ### Fixed @@ -242,6 +243,7 @@ Initial package release. [#117]: https://github.com/JuliaComputing/JuliaHub.jl/issues/117 [#124]: https://github.com/JuliaComputing/JuliaHub.jl/issues/124 [#127]: https://github.com/JuliaComputing/JuliaHub.jl/issues/127 +[#130]: https://github.com/JuliaComputing/JuliaHub.jl/issues/130 [#132]: https://github.com/JuliaComputing/JuliaHub.jl/issues/132 [#133]: https://github.com/JuliaComputing/JuliaHub.jl/issues/133 [#149]: https://github.com/JuliaComputing/JuliaHub.jl/issues/149 diff --git a/docs/src/reference/job-submission.md b/docs/src/reference/job-submission.md index 218137990e..b943ea535f 100644 --- a/docs/src/reference/job-submission.md +++ b/docs/src/reference/job-submission.md @@ -150,6 +150,33 @@ This `.juliabundleignore` will ignore: You can also have additional `.juliabundleignore` files in subdirectories and they will only apply to those directories and their subdirectories. +#### Negating patterns (`!`) + +Like `.gitignore`, a pattern starting with `!` re-includes files that an earlier +pattern excluded. Patterns are applied top to bottom and the **last matching +pattern wins**. This lets you ignore everything and then add back only what you +need: + +``` +* +!programs/ +!programs/* +!Project.toml +!Manifest.toml +``` + +This bundles only the `programs/` directory (and everything under it) plus the +top-level `Project.toml` and `Manifest.toml`, and ignores everything else. + +!!! note "Re-include the parent directory first" + You cannot re-include a file whose parent directory is still excluded — the + bundler skips excluded directories entirely and never descends into them (the + same limitation `git` has). Re-include the directory itself (e.g. `!programs/`) + before, or together with, its contents (`!programs/*`). + +Lines beginning with `#` are treated as comments, and blank lines are ignored. +Use `\!` or `\#` to match a file whose name literally begins with `!` or `#`. + ### Specifying the job image JuliaHub batch jobs can run in various container images. diff --git a/src/PackageBundler/utils.jl b/src/PackageBundler/utils.jl index 914ab4b9ed..6acb295bc2 100644 --- a/src/PackageBundler/utils.jl +++ b/src/PackageBundler/utils.jl @@ -73,17 +73,35 @@ function is_subpath(dir, subpath) return startswith(norm_subpath, norm_dir) end +# Parse one line of a `.juliabundleignore` file into an ordered rule, mirroring +# `.gitignore` syntax. Returns `nothing` for blank lines and `#` comments. A +# leading `!` marks a negated (re-include) rule; a leading `\#` or `\!` escapes a +# literal `#`/`!`. The remainder is compiled with `Glob.FilenameMatch`, so glob +# and trailing-slash directory patterns behave exactly as before. +function _parse_bundleignore_line(line) + s = strip(line) + (isempty(s) || startswith(s, '#')) && return nothing + negated = false + if startswith(s, '!') + negated = true + s = chop(s; head=1, tail=0) + elseif startswith(s, "\\!") || startswith(s, "\\#") + s = chop(s; head=1, tail=0) + end + return (; negated, pattern=Glob.FilenameMatch(String(s))) +end + function get_bundleignore(file, top) dir = dirname(file) - patterns = Set{Any}() + rules = @NamedTuple{negated::Bool, pattern::Glob.FilenameMatch}[] try while true if isfile(joinpath(dir, ".juliabundleignore")) - union!( - patterns, - Glob.FilenameMatch.(strip.(readlines(joinpath(dir, ".juliabundleignore")))), - ) - return patterns, dir + for line in readlines(joinpath(dir, ".juliabundleignore")) + rule = _parse_bundleignore_line(line) + rule !== nothing && push!(rules, rule) + end + return rules, dir end if dir == dirname(dir) || dir == top break @@ -93,7 +111,7 @@ function get_bundleignore(file, top) catch err @warn "Internal error" exception = (err, catch_backtrace()) end - return patterns, top + return rules, top end """ @@ -139,18 +157,26 @@ function path_filterer(top) return false end - patterns, ignorepath = get_bundleignore(path, top) - - rpath = relpath(path, ignorepath) - - return !( - any(p -> occursin(p, sanitize_windows_path(rpath)), patterns) || - # directories specifically can be excluded by patterns that end with a - # path separator, and to match them in case `path` does not have that - # path separator appended, we append it ourselves before matching - isdir(path) && - any(p -> occursin(p, sanitize_windows_path(joinpath(rpath, ""))), patterns) - ) + rules, ignorepath = get_bundleignore(path, top) + + rel = relpath(path, ignorepath) + rpath = sanitize_windows_path(rel) + # directories can be matched by patterns ending in a path separator, so we + # also test the path with a trailing separator appended. + rpath_dir = sanitize_windows_path(joinpath(rel, "")) + isdirp = isdir(path) + + # gitignore semantics: evaluate rules in file order; the last rule that + # matches decides. A plain (non-negated) rule that matches excludes the + # path; a negated (`!`) rule that matches re-includes it. Paths matched by + # no rule are kept. + included = true + for rule in rules + if occursin(rule.pattern, rpath) || (isdirp && occursin(rule.pattern, rpath_dir)) + included = rule.negated + end + end + return included end end diff --git a/test/packagebundler.jl b/test/packagebundler.jl index 68d4bccc1e..a14c90db95 100644 --- a/test/packagebundler.jl +++ b/test/packagebundler.jl @@ -267,6 +267,104 @@ end end end +@testset "path_filterer negation" begin + # gitignore-style: ignore everything, then re-include specific paths. + root = mktempdir() + write( + joinpath(root, ".juliabundleignore"), + """ +* +!programs/ +!programs/* +!Project.toml +!Manifest.toml +""", + ) + mkpath(joinpath(root, "programs", "appA")) + touch(joinpath(root, "programs", "appA", "main.jl")) + touch(joinpath(root, "programs", "loose.txt")) + mkpath(joinpath(root, "secret")) + touch(joinpath(root, "secret", "data.csv")) + touch(joinpath(root, "Project.toml")) + touch(joinpath(root, "Manifest.toml")) + touch(joinpath(root, "README.md")) + pred = JuliaHub._PackageBundler.path_filterer(root) + + # Re-included: parent dir (so Tar descends), its subdir, deep file, loose file. + @test pred(joinpath(root, "programs")) + @test pred(joinpath(root, "programs", "appA")) + @test pred(joinpath(root, "programs", "appA", "main.jl")) + @test pred(joinpath(root, "programs", "loose.txt")) + @test pred(joinpath(root, "Project.toml")) + @test pred(joinpath(root, "Manifest.toml")) + # Still excluded: unlisted dir and file. + @test !pred(joinpath(root, "secret")) + @test !pred(joinpath(root, "README.md")) + + # Last-matching rule wins (order matters). + root2 = mktempdir() + write( + joinpath(root2, ".juliabundleignore"), + """ +*.log +!keep*.log +keep-tmp.log +""", + ) + touch(joinpath(root2, "app.log")) + touch(joinpath(root2, "keep-final.log")) + touch(joinpath(root2, "keep-tmp.log")) + pred2 = JuliaHub._PackageBundler.path_filterer(root2) + @test !pred2(joinpath(root2, "app.log")) # ignored by *.log + @test pred2(joinpath(root2, "keep-final.log")) # re-included by !keep*.log + @test !pred2(joinpath(root2, "keep-tmp.log")) # re-ignored by later keep-tmp.log + + # Comments and blank lines are inert. + root3 = mktempdir() + write(joinpath(root3, ".juliabundleignore"), "# just a comment\n\n*.tmp\n") + touch(joinpath(root3, "a.tmp")) + touch(joinpath(root3, "b.jl")) + pred3 = JuliaHub._PackageBundler.path_filterer(root3) + @test !pred3(joinpath(root3, "a.tmp")) + @test pred3(joinpath(root3, "b.jl")) +end + +@testset "bundle negation (Tar)" begin + root = mktempdir() + write( + joinpath(root, ".juliabundleignore"), + """ +* +!programs/ +!programs/* +!Project.toml +!Manifest.toml +""", + ) + mkpath(joinpath(root, "programs", "appA")) + write(joinpath(root, "programs", "appA", "main.jl"), "println(1)") + write(joinpath(root, "programs", "notes.txt"), "hi") + mkpath(joinpath(root, "secret")) + write(joinpath(root, "secret", "data.csv"), "x,y") + write(joinpath(root, "Project.toml"), "name = \"X\"\n") + write(joinpath(root, "Manifest.toml"), "") + write(joinpath(root, "README.md"), "readme") + + tb = tempname() + Tar.create(JuliaHub._PackageBundler.path_filterer(root), root, tb) + ex = mktempdir() + Tar.extract(tb, ex) + + # Non-empty and contains exactly the re-included content. + @test isfile(joinpath(ex, "programs", "appA", "main.jl")) + @test isfile(joinpath(ex, "programs", "notes.txt")) + @test isfile(joinpath(ex, "Project.toml")) + @test isfile(joinpath(ex, "Manifest.toml")) + # Excluded paths must not be present. + @test !ispath(joinpath(ex, "secret")) + @test !isfile(joinpath(ex, "README.md")) +end + @testset "cp_skip_dangling_symlinks" begin mktempdir() do src # Regular file