diff --git a/.gitattributes b/.gitattributes index 556322be01b4a8..53ccb9407d57bc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,6 +6,8 @@ *.pm text eol=lf diff=perl *.py text eol=lf diff=python *.bat text eol=crlf +*.png binary +/AGENTS.md conflict-marker-size=32 CODE_OF_CONDUCT.md -whitespace /Documentation/**/*.adoc text eol=lf whitespace=trail,space,incomplete /command-list.txt text eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000000000..b49593339932b2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,105 @@ +name: Bug report +description: Use this template to report bugs. +body: + - type: checkboxes + id: search + attributes: + label: Existing issues matching what you're seeing + description: Please search for [open](https://github.com/git-for-windows/git/issues?q=is%3Aopen) or [closed](https://github.com/git-for-windows/git/issues?q=is%3Aclosed) issue matching what you're seeing before submitting a new issue. + options: + - label: I was not able to find an open or closed issue matching what I'm seeing + - type: textarea + id: git-for-windows-version + attributes: + label: Git for Windows version + description: Which version of Git for Windows are you using? + placeholder: Please insert the output of `git --version --build-options` here + render: shell + validations: + required: true + - type: dropdown + id: windows-version + attributes: + label: Windows version + description: Which version of Windows are you running? + options: + - Windows 8.1 + - Windows 10 + - Windows 11 + - Other + default: 2 + validations: + required: true + - type: dropdown + id: windows-arch + attributes: + label: Windows CPU architecture + description: What CPU Archtitecture does your Windows target? + options: + - i686 (32-bit) + - x86_64 (64-bit) + - ARM64 + default: 1 + validations: + required: true + - type: textarea + id: windows-version-cmd + attributes: + label: Additional Windows version information + description: This provides us with further information about your Windows such as the build number + placeholder: Please insert the output of `cmd.exe /c ver` here + render: shell + - type: textarea + id: options + attributes: + label: Options set during installation + description: What options did you set as part of the installation? Or did you choose the defaults? + placeholder: | + One of the following: + > type "C:\Program Files\Git\etc\install-options.txt" + > type "C:\Program Files (x86)\Git\etc\install-options.txt" + > type "%USERPROFILE%\AppData\Local\Programs\Git\etc\install-options.txt" + > type "$env:USERPROFILE\AppData\Local\Programs\Git\etc\install-options.txt" + $ cat /etc/install-options.txt + render: shell + validations: + required: true + - type: textarea + id: other-things + attributes: + label: Other interesting things + description: Any other interesting things about your environment that might be related to the issue you're seeing? + - type: input + id: terminal + attributes: + label: Terminal/shell + description: Which terminal/shell are you running Git from? e.g Bash/CMD/PowerShell/other + validations: + required: true + - type: textarea + id: commands + attributes: + label: Commands that trigger the issue + description: What commands did you run to trigger this issue? If you can provide a [Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) this will help us understand the issue. + render: shell + validations: + required: true + - type: textarea + id: expected-behaviour + attributes: + label: Expected behaviour + description: What did you expect to occur after running these commands? + validations: + required: true + - type: textarea + id: actual-behaviour + attributes: + label: Actual behaviour + description: What actually happened instead? + validations: + required: true + - type: textarea + id: repository + attributes: + label: Repository + description: If the problem was occurring with a specific repository, can you provide the URL to that repository to help us with testing? \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000000..ec4bb386bcf8a4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 37654cdfd7abcf..7baf31f2c471ec 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,19 @@ -Thanks for taking the time to contribute to Git! Please be advised that the -Git community does not use github.com for their contributions. Instead, we use -a mailing list (git@vger.kernel.org) for code submissions, code reviews, and -bug reports. Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/) +Thanks for taking the time to contribute to Git! + +Those seeking to contribute to the Git for Windows fork should see +http://gitforwindows.org/#contribute on how to contribute Windows specific +enhancements. + +If your contribution is for the core Git functions and documentation +please be aware that the Git community does not use the github.com issues +or pull request mechanism for their contributions. + +Instead, we use the Git mailing list (git@vger.kernel.org) for code and +documentation submissions, code reviews, and bug reports. The +mailing list is plain text only (anything with HTML is sent directly +to the spam folder). + +Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/) to conveniently send your Pull Requests commits to our mailing list. For a single-commit pull request, please *leave the pull request description diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000000..22d5376407abf1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +# especially +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot#enabling-dependabot-version-updates-for-actions + +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3da5326f0ba90a..02dc24671646b3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -169,8 +169,11 @@ jobs: NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest + strategy: + matrix: + arch: [x64, arm64] concurrency: - group: vs-build-${{ github.ref }} + group: vs-build-${{ github.ref }}-${{ matrix.arch }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - uses: actions/checkout@v6 @@ -189,14 +192,14 @@ jobs: uses: microsoft/setup-msbuild@v3 - name: copy dlls to root shell: cmd - run: compat\vcbuild\vcpkg_copy_dlls.bat release + run: compat\vcbuild\vcpkg_copy_dlls.bat release ${{ matrix.arch }}-windows - name: generate Visual Studio solution shell: bash run: | - cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows -DHOST_CPU=${{ matrix.arch }} - name: MSBuild - run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 + run: msbuild git.sln -property:Configuration=Release -property:Platform=${{ matrix.arch }} -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar shell: bash env: @@ -210,7 +213,7 @@ jobs: - name: upload tracked files and build artifacts uses: actions/upload-artifact@v7 with: - name: vs-artifacts + name: vs-artifacts-${{ matrix.arch }} path: artifacts vs-test: name: win+VS test @@ -228,7 +231,7 @@ jobs: - name: download tracked files and build artifacts uses: actions/download-artifact@v8 with: - name: vs-artifacts + name: vs-artifacts-x64 path: ${{github.workspace}} - name: extract tracked files and build artifacts shell: bash @@ -420,7 +423,9 @@ jobs: CI_JOB_IMAGE: ${{matrix.vector.image}} CUSTOM_PATH: /custom runs-on: ubuntu-latest - container: ${{matrix.vector.image}} + container: + image: ${{ matrix.vector.image }} + options: ${{ github.repository_visibility == 'private' && '--pids-limit 16384 --ulimit nproc=16384:16384 --ulimit nofile=32768:32768' || '' }} steps: - name: prepare libc6 for actions if: matrix.vector.jobname == 'linux32' diff --git a/.github/workflows/monitor-components.yml b/.github/workflows/monitor-components.yml new file mode 100644 index 00000000000000..f15ff218d28b81 --- /dev/null +++ b/.github/workflows/monitor-components.yml @@ -0,0 +1,94 @@ +name: Monitor component updates + +# Git for Windows is a slightly modified subset of MSYS2. Some of its +# components are maintained by Git for Windows, others by MSYS2. To help +# keeping the former up to date, this workflow monitors the Atom/RSS feeds +# and opens new tickets for each new component version. + +on: + schedule: + - cron: "23 8,11,14,17 * * *" + workflow_dispatch: + +env: + CHARACTER_LIMIT: 5000 + MAX_AGE: 7d + +jobs: + job: + # Only run this in Git for Windows' fork + if: github.event.repository.owner.login == 'git-for-windows' + runs-on: ubuntu-latest + permissions: + issues: write + strategy: + matrix: + component: + - label: git + feed: https://github.com/git/git/tags.atom + - label: git-lfs + feed: https://github.com/git-lfs/git-lfs/tags.atom + - label: git-credential-manager + feed: https://github.com/git-ecosystem/git-credential-manager/tags.atom + - label: tig + feed: https://github.com/jonas/tig/tags.atom + - label: cygwin + feed: https://github.com/cygwin/cygwin/releases.atom + title-pattern: ^(?!.*newlib) + - label: msys2-runtime-package + feed: https://github.com/msys2/MSYS2-packages/commits/master/msys2-runtime.atom + - label: msys2-runtime + feed: https://github.com/msys2/msys2-runtime/commits/HEAD.atom + aggregate: true + - label: openssh + feed: https://github.com/openssh/openssh-portable/tags.atom + - label: libfido2 + feed: https://github.com/Yubico/libfido2/tags.atom + - label: libcbor + feed: https://github.com/PJK/libcbor/tags.atom + - label: openssl + feed: https://github.com/openssl/openssl/tags.atom + title-pattern: ^(?!.*alpha) + - label: gnutls + feed: https://gnutls.org/news.atom + - label: heimdal + feed: https://github.com/heimdal/heimdal/tags.atom + - label: git-sizer + feed: https://github.com/github/git-sizer/tags.atom + - label: gitflow + feed: https://github.com/petervanderdoes/gitflow-avh/tags.atom + - label: curl + feed: https://github.com/curl/curl/tags.atom + title-pattern: ^(?!rc-) + - label: mintty + feed: https://github.com/mintty/mintty/releases.atom + - label: 7-zip + feed: https://sourceforge.net/projects/sevenzip/rss?path=/7-Zip + aggregate: true + - label: bash + feed: https://git.savannah.gnu.org/cgit/bash.git/atom/?h=master + aggregate: true + - label: perl + feed: https://github.com/Perl/perl5/tags.atom + title-pattern: ^(?!.*(5\.[0-9]+[13579]|RC)) + - label: pcre2 + feed: https://github.com/PCRE2Project/pcre2/tags.atom + - label: mingw-w64-llvm + feed: https://github.com/msys2/MINGW-packages/commits/master/mingw-w64-llvm.atom + - label: innosetup + feed: https://github.com/jrsoftware/issrc/tags.atom + - label: mimalloc + feed: https://github.com/microsoft/mimalloc/tags.atom + title-pattern: ^(?!v1\.|v3\.[01]\.) + fail-fast: false + steps: + - uses: git-for-windows/rss-to-issues@v0 + with: + feed: ${{matrix.component.feed}} + prefix: "[New ${{matrix.component.label}} version]" + labels: component-update + github-token: ${{ secrets.GITHUB_TOKEN }} + character-limit: ${{ env.CHARACTER_LIMIT }} + max-age: ${{ env.MAX_AGE }} + aggregate: ${{matrix.component.aggregate}} + title-pattern: ${{matrix.component.title-pattern}} diff --git a/.github/workflows/nano-server.yml b/.github/workflows/nano-server.yml new file mode 100644 index 00000000000000..a9cf026efeb2a6 --- /dev/null +++ b/.github/workflows/nano-server.yml @@ -0,0 +1,76 @@ +name: Windows Nano Server tests + +on: + workflow_dispatch: + +env: + DEVELOPER: 1 + +jobs: + test-nano-server: + runs-on: windows-2022 + env: + WINDBG_DIR: "C:/Program Files (x86)/Windows Kits/10/Debuggers/x64" + IMAGE: mcr.microsoft.com/powershell:nanoserver-ltsc2022 + + steps: + - uses: actions/checkout@v6 + - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - name: build Git + shell: bash + run: make -j15 + - name: pull nanoserver image + shell: bash + run: docker pull $IMAGE + - name: run nano-server test + shell: bash + run: | + docker run \ + --user "ContainerAdministrator" \ + -v "$WINDBG_DIR:C:/dbg" \ + -v "$(cygpath -aw /mingw64/bin):C:/mingw64-bin" \ + -v "$(cygpath -aw .):C:/test" \ + $IMAGE pwsh.exe -Command ' + # Extend the PATH to include the `.dll` files in /mingw64/bin/ + $env:PATH += ";C:\mingw64-bin" + + # For each executable to test pick some no-operation set of + # flags/subcommands or something that should quickly result in an + # error with known exit code that is not a negative 32-bit + # number, and set the expected return code appropriately. + # + # Only test executables that could be expected to run in a UI + # less environment. + # + # ( Executable path, arguments, expected return code ) + # also note space is required before close parenthesis (a + # powershell quirk when defining nested arrays like this) + + $executables_to_test = @( + ("C:\test\git.exe", "", 1 ), + ("C:\test\scalar.exe", "version", 0 ) + ) + + foreach ($executable in $executables_to_test) + { + Write-Output "Now testing $($executable[0])" + &$executable[0] $executable[1] + if ($LASTEXITCODE -ne $executable[2]) { + # if we failed, run the debugger to find out what function + # or DLL could not be found and then exit the script with + # failure The missing DLL or EXE will be referenced near + # the end of the output + + # Set a flag to have the debugger show loader stub + # diagnostics. This requires running as administrator, + # otherwise the flag will be ignored. + C:\dbg\gflags -i $executable[0] +SLS + + C:\dbg\cdb.exe -c "g" -c "q" $executable[0] $executable[1] + + exit 1 + } + } + + exit 0 + ' diff --git a/.gitignore b/.gitignore index 4da58c6754899e..c730b1587e518a 100644 --- a/.gitignore +++ b/.gitignore @@ -172,6 +172,7 @@ /git-submodule /git-submodule--helper /git-subtree +/git-survey /git-svn /git-switch /git-symbolic-ref @@ -259,5 +260,6 @@ Release/ /git.VC.db *.dSYM /contrib/buildsystems/out +CMakeSettings.json /contrib/libgit-rs/target /contrib/libgit-sys/target diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000000..377e9e93724c18 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,1202 @@ +# Git for Windows - Development Guide + +## Background + +Git for Windows is a fork of upstream Git that provides the necessary +adaptations to make Git work well on Windows. While the primary target is +Windows, the project also maintains working builds on other platforms (Linux, +macOS) because cross-platform builds often catch mistakes that might be missed +when testing only on Windows. + +There are downstream projects that build on Git for Windows, such as Microsoft +Git, which adds features for large monorepos hosted on Azure DevOps. + +## Overview + +This document provides guidance for developing and debugging in +Git for Windows. + +## Repository Structure + +### Branch Naming Patterns + +Based on actual repository usage: + +- `main` - The primary development branch +- Feature branches use descriptive topic names, targeting the main branch + +## Building and Testing + +### Build + +```bash +make -j$(nproc) +``` + +On Windows (in a Git for Windows SDK shell): + +```bash +make -j15 +``` + +### Run Specific Tests + +```bash +cd t && sh t0001-init.sh # Run normally +cd t && sh t0001-init.sh -v # Verbose +cd t && sh t0001-init.sh -ivx # verbose, trace, fail-fast +``` + +Some tests are expensive and skipped by default. When a test exits immediately +with "skip all", check the test script header for `test_bool_env GIT_TEST_*` +to find which environment variable enables it. + +## Git Source Code Structure + +This section provides a bird's eye view of Git's source code layout. For +more details, see "A birds-eye view of Git's source code" in +`Documentation/user-manual.adoc`. + +### Key Directories + +| Directory | Purpose | +|------------------|----------------------------------------------------| +| `builtin/` | Built-in command implementations (`cmd_()`) | +| `xdiff/` | Low-level diff algorithms (libxdiff) | +| `t/` | Test suite (shell scripts, helpers, libraries) | +| `Documentation/` | Man pages, guides, technical docs (AsciiDoc) | +| `contrib/` | Optional extras, not part of core Git | +| `compat/` | Platform compatibility shims | +| `refs/` | Reference backends (files, reftable) | +| `reftable/` | Reftable format implementation | + +### Built-in Commands + +Built-in commands are implemented in `builtin/.c` with a function +`cmd_()`. To add a new built-in: + +1. Create `builtin/.c` implementing `cmd_()` +2. Add entry to the `commands[]` array in `git.c`: + ```c + { "", cmd_, RUN_SETUP }, + ``` +3. Add to `BUILTIN_OBJS` in `Makefile` +4. Add to `command-list.txt` with appropriate category +5. Run `make check-builtins` to verify consistency + +### Object Data Model + +Git stores four types of objects, defined in `object.h`: + +```c +enum object_type { + OBJ_COMMIT = 1, /* Points to tree, has parent commits, metadata */ + OBJ_TREE = 2, /* Directory listing: names -> blob/tree OIDs */ + OBJ_BLOB = 3, /* File contents */ + OBJ_TAG = 4, /* Annotated tag pointing to another object */ +}; +``` + +Objects are addressed by their SHA (OID) and stored in the Object Database. + +### Object Database (ODB) + +The ODB is defined in `odb.h` and implemented in `odb.c`: + +- **`struct object_database`**: Top-level container, owned by a repository + - `sources`: Linked list of `odb_source` (primary + alternates) + - `replace_map`: Object replacements (see `git-replace(1)`) + - `commit_graph`: Commit-graph cache for faster traversal + +- **`struct odb_source`**: A single object store location + - `path`: Directory (e.g., `.git/objects` or an alternate) + - `loose`: Loose object cache + - `packfiles`: Packfile store (idx + pack files) + +Key functions: +- `odb_read_object()`: Read an object by OID +- `odb_write_object()`: Write an object, returns OID +- `odb_read_object_info()`: Get object type/size without reading content + +### Documentation + +Documentation lives in `Documentation/` as AsciiDoc (`.adoc`) files: + +- `git-.adoc` - Man pages for commands +- `config/.adoc` - Config option documentation (included by others) +- `technical/` - Technical specifications and internals + +To build documentation: +```bash +make -C Documentation html # Build HTML docs +make -C Documentation man # Build man pages +``` + +To add documentation for a new config option, add it to the appropriate +file in `Documentation/config/`. These are included by other docs. + +To lint documentation: +```bash +make -C Documentation lint-docs +``` + +## Debugging Techniques + +### Debugging Philosophy + +Debugging is not about guessing fixes and seeing if they work. It is about +building a complete understanding of the problem before attempting any fix. +The goal is not speed to a "fix" but confidence that you understand and have +addressed the root cause. + +**Respect turnaround time.** If seeing the result of an attempted fix takes +7-10 minutes (e.g., a CI workflow run), you cannot afford to guess. Each +iteration costs human time and attention. Before pushing any change: + +1. Ask: "What information am I missing to competently assess this situation?" +2. Add diagnostic output that will provide that information if the fix fails. +3. Consider whether you can reproduce the issue locally where turnaround is + seconds, not minutes. + +**Understand before acting.** Before attempting any fix: + +1. When investigating a regression between two versions, start by examining + the code diff. Analyze what actually changed before running any tests. + Tests confirm hypotheses; reading the diff gives you the hypothesis. +2. Trace the code flow completely. Read the relevant Makefiles, scripts, and + source files. Understand what each component does and how they interact. +3. Identify all changes that could have contributed: upstream commits, + downstream patches, infrastructure changes (CI runner updates, dependency + upgrades). +4. For each potential cause, find the specific commit, its date, its intent, + and how it interacts with other components. +5. Build a hypothesis. Then ask: "How would I confirm or disprove this?" + +**Do not assume root cause from symptoms.** A symptom appearing on one +platform does not mean the bug is platform-specific. The cause may be in +shared code that manifests differently across platforms. Similarly, a passing +test on one platform when it fails on another is data to investigate, not +grounds to conclude "works for me." + +**When a fix does not work, investigate why.** If you expected a fix to work +and it did not, that is valuable information. Do not abandon that line of +thinking and try something else. Instead: + +1. Ask: "Why didn't that work? What does this tell me about my understanding?" +2. Add more targeted diagnostics to understand the discrepancy. +3. Re-examine your assumptions. Something you believed to be true is false. + +**Add diagnostics proactively.** Before pushing a fix attempt, add diagnostic +output that will: + +1. Confirm the state you expect to see if the fix works. +2. Reveal the actual state if it does not. +3. Provide enough context to understand the next step without another round + trip. + +For build failures, this might include: library paths, compiler flags, +architecture information, symbol tables, file existence checks, environment +variables. + +**Build confidence before pushing.** A fix should not be a guess. You should +be able to explain: + +1. What was the root cause? +2. Why does this fix address it? +3. What other ways could this problem be solved? +4. Am I choosing the "most correct" or "most effective" approach? +5. What evidence confirms your understanding? +6. What could still go wrong, and how would you detect it? + +### Searching the Codebase + +In particular when debugging failures that printed error messages, it is often +a useful thing to search for those error messages; If parts of the message seem +mutable (e.g. commit OIDs), those will not be hard-coded and the search needs +to accommodate for that by using regular expressions or prefix matches. + +Use `git grep` for fast code searches: + +```bash +git grep -n -i "pattern" # Case-insensitive search with line numbers +git grep -n -w "word" # Whole-word matches only +git grep -n -i "pattern" -- "*.c" # Search only C files +``` + +### Trace2 + +Enable tracing to see command execution patterns: +```bash +GIT_TRACE2_EVENT=/path/to/trace.txt git +``` + +### Instrumenting Git Internals During Tests + +When adding debug output to Git's C code during test investigation, +`fprintf(stderr, ...)` from git subprocesses spawned by the test framework +is typically swallowed (redirected or discarded by the test harness). Use +Trace2 instead: + +```c +trace2_data_intmax("index", NULL, "my_debug/cache_nr", istate->cache_nr); +trace2_data_string("index", NULL, "my_debug/state", some_string); +``` + +Then run the test with `GIT_TRACE2_EVENT` or `GIT_TRACE2_PERF` pointing to +a file, and grep the output. This integrates with Git's existing tracing +infrastructure and survives the test framework's output management. + +As a last resort (e.g. when Trace2 is not initialized yet at the point you +need to instrument), write to a fixed file path: + +```c +FILE *f = fopen("/tmp/debug.log", "a"); +if (f) { fprintf(f, "state: %u\n", value); fclose(f); } +``` + +### Comparing Branches After Rebase + +```bash +# See what patches exist in a new branch but not old +git log --oneline old-branch..new-branch +# or +git range-diff -s --right-only old-branch...new-branch + +# Compare specific files between branches +git diff old-branch..new-branch -- path/to/file.c +# or +git log -p old-branch..new-branch -- path/to/file.c +# or even +git log -L start-line,end-line:path/to/file.c old-branch..new-branch -- + +# Find upstream changes between tags +git log --oneline --first-parent v2.52.0..v2.53.0 +``` + +### Test Failure Investigation + +1. **Reproduce with tracing**: Run test with `-ivx` flags +2. **Check timestamps**: Look at `t_abs` in trace to understand ordering +3. **Compare with working version**: Build and test the previous version +4. **Bisect if needed**: Use `git bisect` to find the breaking commit + +Bisecting failures introduced by upstream commits require some stunts to +apply the downstream changes for every bisection step. This can be done by +squashing all downstream changes into one throw-away commit and then +cherry-picking that (typically, there will be merge conflicts the farther +away from the original branch point the commit is cherry-picked to, so it +often makes sense to squash both old and new downstream changes, and then +to "interpolate" between them when encountering merge conflicts). + +### Bisecting Failures in `seen` + +When a topic passes on its own but fails after being merged to `seen`, the +failure is caused by interaction with another in-flight topic. To identify +the culprit: + +1. Fetch the exact `seen` commit from the failing CI run (get the SHA from + the workflow run metadata via the GitHub API). +2. Use a worktree checked out at that `seen` commit. +3. Bisect the first-parent history between `upstream/master` and `seen~1` + (excluding the topic's own merge). At each bisection step, merge the + topic in temporarily, build, run the test, then undo the merge. +4. Write a `git bisect run` script that automates this. Key pitfalls: + - The script must `unset` test environment variables (especially + `GIT_TEST_SPLIT_INDEX`) before cleanup operations like + `git checkout -f`, otherwise the worktree's own index can get + corrupted. + - Use `git checkout -f "$ORIG"` (not `git reset --hard`) to undo the + temporary merge, since `reset --hard` under split-index can corrupt. + - Save the current commit OID at the start (`ORIG=$(git rev-parse HEAD)`) + because `ORIG_HEAD` is unreliable during bisect. + - On merge conflict, return 125 (skip) and `git merge --abort`. +5. Store the alias for running with the full set of CI test variables as a + repository-local alias (to avoid repeating the long export list and to + allow the user to approve the tool call once). + +### CI/Workflow Failure Investigation + +When a CI workflow fails, the debugging process has a high cost per iteration. +Approach these failures methodically: + +**1. Establish what changed.** Before looking at the error, identify: + +- What was the last successful run? What version/commit was it based on? +- What changed between then and now? (upstream commits, downstream patches, + runner image updates, dependency changes) +- Use the GitHub API to retrieve run metadata and compare. + +**2. Analyze the error deeply.** Read the full error message and surrounding +context. Understand: + +- What command failed? +- What were its inputs (flags, environment, paths)? +- What did it expect vs. what did it get? + +**3. Trace the code flow locally.** Before making any CI changes: + +- Read the workflow YAML, Makefiles, and scripts involved. +- Understand how variables flow from one to another. +- Identify where the failing values come from. + +**4. Reproduce locally if possible.** Many CI failures can be reproduced +locally with faster turnaround: + +- For build failures: replicate the build environment and commands. +- For macOS issues: if you lack a Mac, at least trace the Makefile logic + to understand what flags should be set and why. +- For test failures that only appear in specific CI jobs (like + `linux-TEST-vars`): reproduce with the _exact_ set of environment + variables that job sets. Check `ci/run-build-and-tests.sh` for the + job's variable block. Do not assume a single variable (e.g. + `GIT_TEST_SPLIT_INDEX`) is sufficient; other variables may contribute + to the failure path. +- When a test fails in `seen` but not on the topic branch alone, check + out the exact `seen` commit from the failing CI run (get the SHA from + the workflow run metadata) and reproduce against that. The interaction + with other in-flight topics is the likely cause. + +**5. Do not assume CI coverage from platform support.** When asking "why +does platform X not see this bug?", verify whether CI actually tests that +combination on that platform. For example, `GIT_TEST_SPLIT_INDEX=yes` is +only set by `linux-TEST-vars`; there is no equivalent `osx-TEST-vars` or +`windows-TEST-vars` job. A bug that only manifests under split-index +testing may be present on all platforms but only caught on Linux. + +**5. Add comprehensive diagnostics on first attempt.** If you must push to +CI to test, make that push count: + +- Add diagnostic output for every hypothesis you have. +- Print the values of key variables, paths, flags. +- Show the state before and after key operations. +- Design diagnostics to distinguish between your hypotheses. + +**6. Do not remove diagnostics until the problem is solved.** Keep them in +"drop!" commits so they can be easily removed later but provide information +if subsequent fixes also fail. + +**7. When a fix fails, treat it as data.** The failure tells you something. +Your mental model was wrong. Figure out what before trying again. + +## Git Workflow + +This repository is a shared development environment, not a sandbox. Exercise +caution with all Git operations. + +### Committing Changes + +Never use `git add -A` or `git add .` - these commands will stage untracked +build artifacts, editor swap files, and other detritus that should not be +committed. Always specify pathspecs explicitly: + +```bash +# Good: stage and commit specific files +git commit -sm "your message here" path/to/file.c other/file.h + +# Bad: stages everything, including untracked garbage +git add -A && git commit -m "message" +``` + +The `-s` flag adds a Signed-off-by trailer, which is required for this +project. + +When AI assistance is used to author or co-author a commit, add a +Co-authored-by trailer identifying the model: + +```bash +git commit -s --trailer "Co-authored-by: " -m "message" file.c +``` + +### Pushing Changes + +Never push without explicit user permission. The user controls when and +where changes are pushed. This is especially critical because: + +- The repository has multiple remotes with different purposes +- Force-pushing to the wrong remote can cause significant damage +- Tags require special handling (`git push --tags` or explicit tag pushes) + +Wait for the user to push, or ask explicitly before pushing. + +### Making Code Changes + +**Minimal, surgical changes.** Make the smallest possible change to achieve +the goal. Do not rewrite entire files or functions when a targeted edit +suffices. When removing functionality: + +1. Remove the code paths that invoke the unwanted functionality +2. Compile to identify what is now unused +3. Remove the unused functions one at a time +4. Repeat until clean + +**No fly-by changes.** Do not make changes that were not requested, even if +they seem like improvements (renaming variables, reformatting untouched code, +"fixing" things not part of the task). If you believe a change would be +beneficial but it was not requested, ask for permission first. + +**The human is the driver.** Execute what is asked. If you think something +should be done differently, ask---do not just do it. + +### Commit Message Quality + +Good commit messages use flowing English prose, not bullet points. They +clearly state: + +- **Context**: What situation prompted this change? Include URLs to failing + CI runs, issue numbers, or other references that future readers will need. +- **Intent**: What is this change trying to accomplish? +- **Justification**: Why is this the right approach? What alternatives were + considered? When choosing between approaches based on performance, + include measured timings so future readers understand the tradeoffs. +- **Implementation**: How does the change work? (Only for non-obvious parts; + don't describe what's clear from the diff.) + +Include exact error messages rather than vague descriptions. If a build +failed with `Undefined symbols for architecture arm64: "_iconv"`, put that +in the commit message - don't just say "fixed a linker error." + +Wrap commit messages at 76 columns per line. + +### Commit Prefixes for Rebase Workflows + +This repository uses interactive rebase with autosquash. Commit prefixes +signal intent: + +- **`fixup! `**: Will be squashed into the referenced commit + during rebase. The title after `fixup!` must match the original commit's + title exactly. +- **`drop!`**: Indicates a commit that should be dropped before the final + merge. Used for debugging, temporary workarounds, or experiments. + +To find the correct title for a fixup commit: + +```bash +git log --oneline path/to/changed/file | head -10 +``` + +Then use the exact title: + +```bash +git commit -sm "fixup! release: add Mac OSX installer build" path/to/file +``` + +## Rebasing Workflow + +Rebases are the bread and butter of Git for Windows: topic branches are +rebased every time upstream Git releases a new version. This section covers +the workflow for managing downstream patches through repeated rebases. + +### Merging-Rebases + +Git for Windows uses "merging-rebases" to maintain downstream patches. Unlike +a flat series of commits, the downstream changes are organized as topic +branches merged together, preserving the logical grouping of related changes. + +Each integration branch (`main`, `shears/next`, `shears/seen`) contains a +marker commit with the message "Start the merging-rebase to \". This +commit separates upstream history from downstream patches. Reference it with: + +```bash +# Find the marker commit +git log --oneline --grep="Start the merging-rebase" -1 + +# Reference it using commit message search syntax +origin/main^{/Start.the.merging-rebase} +``` + +When working with merging-rebases: + +- **Downstream patches start after the marker**: Use + `origin/main^{/Start.the.merging-rebase}..origin/main` to see all + downstream commits +- **Topic branches are merged, not rebased flat**: Each logical feature or + fix is a branch merged into the integration branch +- **Merge commits are preserved**: The rebase recreates the merge structure + on top of the new upstream base + +To compare downstream patches before and after a rebase: + +```bash +# Compare the old and new downstream patch series +git range-diff \ + old-base^{/Start.the.merging-rebase}..old-branch \ + new-base^{/Start.the.merging-rebase}..new-branch +``` + +### Starting a Merging-Rebase + +To rebase the downstream patches onto a new upstream version, create a marker +commit and use it as the base for an interactive rebase: + +```bash +# Variables for the commit message +tag=v2.53.0 +# The previous marker - this becomes the exclusion point for --onto +previousMergeOid=$(git rev-parse origin/main^{/Start.the.merging-rebase}) +tagOid=$(git rev-parse "$tag") +tipOid=$(git rev-parse origin/main) + +# Create the marker commit with two parents: the tag and the current tip +markerOid=$(git commit-tree "$tag^{tree}" -p "$tag" -p "$tipOid" -m "Start the merging-rebase to $tag + +This commit starts the rebase of $previousMergeOid to $tagOid") + +# Graft the marker to appear as if it has only the tag as parent +git replace --graft "$markerOid" "$tag" + +# Use the marker as the base for rebasing (only commits after previousMergeOid) +git rebase -r --onto "$markerOid" "$previousMergeOid" origin/main + +# After the rebase completes, delete the replace ref +git replace -d "$markerOid" +``` + +The marker commit is created with two parents: the upstream tag and the +current branch tip. The `git replace --graft` makes Git see only the tag as +parent during the rebase, allowing the downstream commits to be cleanly +rebased onto the new upstream. After the rebase completes, the replace ref +is deleted to clean up. + +#### The shears/* Branches + +Upstream Git has four integration branches: `seen`, `next`, `master`, and +`maint`. Git for Windows maintains a corresponding `shears/*` branch for each +(`shears/seen`, `shears/next`, `shears/master`, `shears/maint`) that +continuously rebases Git for Windows' `main` onto the respective upstream +branch. + +These branches are updated incrementally rather than from scratch, avoiding +re-resolution of merge conflicts. The update process leverages reachability: + +1. **Integrate new downstream commits**: If `origin/main` has commits not yet + in the shears branch, rebase them on top (using `-r` to preserve branch + structure). Update the marker commit's message and second parent. + +2. **Integrate new upstream commits**: If the upstream branch has commits not + yet integrated, rebase onto the new upstream tip. Update the marker commit + accordingly. + +The marker commit's second parent always points to the current `origin/main` +tip, making it trivial to identify what downstream commits are included. +Similarly, the marker's first parent (the upstream base) shows exactly which +upstream version is integrated. + +### When to Skip a Patch + +Use `git rebase --skip` when the patch is already in the new base: + +- **Upstreamed**: The patch was accepted upstream and is now in `seen` +- **Backported**: A fix we backported is now included in the upstream base +- **Superseded**: HEAD already contains evolved code that includes this + change + +Signs to skip rather than resolve: HEAD has the functionality, the +conflict would discard the patch entirely, or `git range-diff` shows +the downstream and upstream patches are equivalent. + +To find the corresponding upstream commit for a conflicting patch: + +```bash +git range-diff --left-only REBASE_HEAD^! REBASE_HEAD.. +``` + +### Resolving Merge Conflicts + +When resolving merge conflicts during a rebase (especially when squashing +fixups), the goal is to **apply the minimal surgical change** that the +patch intended, not to reconstruct entire functions or add duplicate code. + +#### 1. Understand What the Patch Wants + +First, examine the patch being applied: + +```bash +git show REBASE_HEAD +``` + +Look at the actual changes (lines starting with `-` and `+`): +- What lines are being removed? +- What lines are being added? +- What is the context (function name, nearby code)? + +**Key insight**: The patch shows the *intent*---a specific small change to +make. Focus on this, not on the conflict markers' content. + +**Code movement detection**: If the patch shows large changes, check with +`--ignore-space-change`: + +```bash +git show --ignore-space-change +``` + +This reveals whether the commit is primarily **moving code** (lots of +whitespace changes) or making **logic changes** (actual code modifications). +When code was moved and re-indented, focus only on the non-whitespace +changes when resolving the conflict. + +#### 2. Understand Where the Code Is Now + +The conflict occurred because the code moved or changed since the patch was +created. Find where that code actually exists now: + +```bash +# If the patch was changing a specific pattern, find all occurrences +git grep -n "pattern from patch" + +# View the conflicted file around those locations +``` + +**Common mistake**: Assuming the conflict markers show you what to do. They +do not---they just show where Git got confused. + +#### 3. Apply the Surgical Change + +Make **only** the change the patch intended, but in the current location: + +- If the patch adds `--abbrev=12` to a range-diff call, find where that + range-diff call is NOW and add it there +- If the patch changes a `.split()` pattern, find where that pattern is NOW + and change it +- Do not copy entire functions from the conflict markers +- Do not create duplicates + +#### 4. Remove ALL Conflict Markers + +Conflict markers make the file invalid code: +``` +<<<<<<< HEAD +======= +>>>>>>> commit-hash +``` + +**All three types of markers must be completely removed.** + +#### 5. Verify the Resolution + +**Critical**: After staging your resolution, verify it matches the patch +intent: + +```bash +# Compare your staged changes to the original patch +git diff --cached +git rebase --show-current-patch + +# Or more directly, compare to REBASE_HEAD +git diff --cached +git show REBASE_HEAD + +# For code that was moved/re-indented, ignore whitespace +git diff --cached --ignore-space-change +git show REBASE_HEAD --ignore-space-change +``` + +**Verify, verify, verify**: The output of `git diff --cached` should +correspond closely to the diff in `git show REBASE_HEAD`. The line numbers +and context will differ (because code moved), but the actual changes (the +`-` and `+` lines) should match the patch intent. + +**After completing a rebase**, always verify the final result: + +```bash +# Compare tree before and after rebase +git diff @{1} + +# Shows what changed in each rebased commit +git range-diff @{1}... +``` + +If the rebase was onto the same base commit (e.g., squashing fixups), the +`git diff @{1}` should be empty---this proves the rebase only reorganized +commits without changing the end result. If the rebase was onto a new base +commit (e.g., rebasing onto a new upstream release), the diff should match +the difference between the old and new base commits, modulo any changes +from upstreamed or backported patches. The `git range-diff @{1}...` shows +the intended amendments (like adding `--abbrev=12`) were correctly applied +to each commit. + +### Conflict Resolution Red Flags + +These indicate you are doing it wrong: + +- Your diff adds hundreds of lines when the patch only changed 3 +- Conflict markers remain in the file +- Functions appear twice in the file +- You added `<<<<<<< HEAD` or `=======` to the staged changes +- Syntax check fails after resolution + +### Key Conflict Resolution Lessons + +1. **Context changes, intent does not** - The patch's line numbers are + wrong, but the change is right +2. **Conflict markers lie** - They show you where Git got confused, not + what you should do +3. **One change at a time** - If the patch adds one line, your resolution + should add one line +4. **Verify, verify, verify** - `git diff --cached` should match + `git show REBASE_HEAD` (modulo context) +5. **Post-rebase verification** - `git diff @{1}` (empty) and + `git range-diff @{1}...` (shows amendments) +6. **Ignore whitespace for code moves** - Use `--ignore-space-change` to + see the actual logic changes when code was moved and re-indented +7. **When in doubt, look at the range-diff** - `git range-diff` shows if + you matched the intent + +### Useful Rebase Tools + +- `git rebase --show-current-patch` - See what change is being applied +- `git show REBASE_HEAD` - Alternative to above, works better with + `--ignore-space-change` +- `git show --ignore-space-change` - See only logic changes, not + whitespace/indentation +- `git grep -n "pattern"` - Find where code moved to +- `git log -L ,: REBASE_HEAD..HEAD` - See how upstream + modified a line range since the original patch; invaluable for + understanding how conflicting lines changed +- `git diff --cached` - After staging resolution, verify it matches + REBASE_HEAD +- `git diff @{1}` - After rebase, compare tree before/after +- `git range-diff @{1}...` - After rebase, verify intended changes were made +- `git range-diff A^! B^!` - Compare original patch to your resolution + +### Leveraging Rerere + +Git's "reuse recorded resolution" (`rerere`) feature automatically records +how you resolve conflicts and replays those resolutions when the same +conflict recurs. This is invaluable for repeated rebases where the same +downstream patches conflict with similar upstream changes. + +When you see `Staged 'file' using previous resolution`, Git has applied a +previously recorded resolution. Always verify these auto-resolutions are +still correct---upstream context may have changed enough that the old +resolution no longer applies cleanly. + +To enable rerere: +```bash +git config --global rerere.enabled true +``` + +### Automation Tips + +When running rebases in automated or scripted contexts, disable the pager +to avoid hangs: + +```bash +GIT_PAGER=cat git range-diff ... +# or +git --no-pager log ... +``` + +### Non-interactive "Interactive" Rebases + +AI agents cannot drive interactive editors reliably. Instead, insert a +`break` as the first todo command so the rebase stops immediately, then +edit the todo file directly: + +```bash +# Start the rebase, stopping before any picks execute +GIT_SEQUENCE_EDITOR='sed -i 1ib' git rebase -ir + +# Find and edit the todo file with the view/edit tools +git rev-parse --git-path rebase-merge/git-rebase-todo + +# After editing the todo, continue (GIT_EDITOR=true suppresses the +# editor that fixup -C and amend! commands would otherwise open) +GIT_EDITOR=true git rebase --continue +``` + +### Scripted Hunk Staging + +`git add -p` is interactive by default, but its prompts follow a +predictable protocol. To stage the first hunk of a file without +human interaction: + +```bash +printf '%s\n' s y q | git add -p +``` + +The `s` splits a large hunk, `y` stages the first sub-hunk, and `q` +quits. Adjust the sequence for different hunk selections (e.g., +`y y n q` to stage the first two hunks but skip the third). + +### Finding Which Commit to Amend + +When a working-tree change belongs in an earlier commit (an `hg absorb` +workflow), use `git log -L` to find which commit last touched the +relevant lines: + +```bash +git log -L ,+: +``` + +This shows the full history of a line range, making it easy to identify +the commit whose title you need for a `fixup!` commit. This is far more +surgical than grepping through full diffs. + +### Fixup Commits + +Downstream patches sometimes require adjustment due to changes in the +environment they operate in. These changes may come from: + +- **Upstream code changes**: API modifications, struct field moves, + declarations relocating between headers, or semantic changes in functions + that downstream code depends on. +- **External environment changes**: CI runner image updates, toolchain + upgrades, dependency version changes, or platform behavior shifts. + +In both cases, create a `fixup!` commit that will be squashed into the +original downstream patch during the next interactive rebase. The commit +message body must precisely document the change that necessitated the fix: + +- For upstream changes: reference the specific upstream commit (by OID or + title) and explain what it changed. +- For external changes: include URLs to failing CI runs, document what + changed in the environment (e.g., "GitHub Actions macos-latest runner + upgraded from macOS 14 to macOS 15"), and note the exact error message. + +This documentation is essential because the fixup will be squashed away, +and the context will be lost if not recorded in the commit message that +gets squashed into. + +Run affected tests before finalizing. + +### `amend!` Commits + +A `fixup!` commit keeps the target's commit message and merely combines +its diff into the target. An `amend!` commit additionally **replaces** +the target's commit message with its own body. Use `amend!` when the +fix changes the meaning of the target sufficiently that the original +subject or body is no longer accurate, or when the goal is to align a +downstream commit with a specific upstream replacement. + +The format is rigid: the first line of an `amend!` commit must be +exactly `amend! `, followed by a blank line and then +the **new** commit message that should replace the target's, starting +with the new subject line: + +``` +amend! mingw: use mimalloc + +mingw: stop using nedmalloc + +The vendored nedmalloc allocator under compat/nedmalloc/ has been +unmaintained upstream... +``` + +After autosquash, the resulting commit has the new subject (`mingw: +stop using nedmalloc`), the new body, and a diff that is the +composition of the target's diff and the `amend!`'s diff. Crafting the +`amend!` diff so that the composition equals a known upstream commit's +diff is the canonical way to align a downstream branch-thicket commit +with an in-flight upstream replacement: when the next merging-rebase +picks up the upstream commit, the byte-identical downstream commit +collapses into it cleanly. + +### PRs Composed Entirely of `fixup!` and `amend!` Commits + +Adjusting or removing a feature that lives in the branch thicket is +often best expressed as a PR that consists *only* of `fixup!` and +`amend!` commits targeting the existing thicket commits. Each pair +autosquashes during the next merging-rebase. Pairs whose diffs cancel +exactly produce empty commits, which the rebase drops with +`--empty=drop`. The end state is *as if the original commits had been +edited or removed in place*, while preserving review-friendly atomic +patches in the PR. + +This is the preferred pattern for reverting a multi-commit downstream +feature. Order the fixups in **reverse** of the originals so each +revert applies cleanly to the worktree as you build the series. +### Common Adaptation Patterns + +**Struct field moves**: When upstream moves fields between structs, update +all downstream code that accesses those fields. + +**API changes**: When upstream changes function signatures, update callers +and verify semantics are preserved. + +**New abstractions**: When upstream introduces new layers, ensure downstream +code uses the correct instance. + +## Coding Conventions + +The Git project maintains a charmingly old-school, Unix-greybeard aesthetic +when it comes to text encoding. In the spirit of the PDP-11 and Bell Labs +terminal sessions of yore: + +- **ASCII only**: Avoid Unicode characters in source code, comments, and + documentation. Use `->` instead of `→`, `--` instead of `—`, and so on. + To verify your changes contain no non-ASCII characters: + ``` + git diff | LC_ALL=C grep '[^ -~]' + ``` +- **80 columns per line**: The mailing list veterans will "kindly" remind you + that lines should not exceed 80 characters (they do mean columns, but + let's not split beards or hairs about wide glyphs). + First, check for whitespace errors (trailing whitespace, mid-line tabs, etc.): + ``` + git diff --check + ``` + Once that passes, you know tabs only appear at line beginnings, so each + tab equals exactly 8 columns. To find lines exceeding 80 columns: + ``` + git diff --no-color | grep '^+' | sed 's/\t/ /g' | grep '.\{82\}' + ``` + (We use 82 because diff output prefixes added lines with `+`.) +- **Tabs for indentation**: The codebase uses tabs, not spaces. +- **No trailing whitespace**: Clean up your lines. + +**Pre-commit checklist.** Run all three checks before every commit: + +```bash +git diff --check && +git diff --no-color | LC_ALL=C grep '[^ -~]' && + echo "ERROR: non-ASCII characters found" && +git diff --no-color | grep '^+' | sed 's/\t/ /g' | + grep '.\{82\}' && + echo "ERROR: lines exceed 80 columns" +``` + +The first command catches whitespace errors. If either of the latter +two produces output, fix the offending lines before committing. Note +that these checks apply to commit messages as well (wrap at 76 columns +for messages, 80 for code). + +See `Documentation/CodingGuidelines` for the full set of conventions. + +### strbuf patterns + +Use `strbuf_addf()` with string continuation for multi-line content instead +of multiple `strbuf_addstr()` calls: + +```c +/* Good */ +strbuf_addf(&buf, + "tree %s\n" + "author %s\n" + "committer %s\n" + "\ncommit message\n", + tree_hex, author, committer); + +/* Avoid */ +strbuf_addstr(&buf, "tree "); +strbuf_addstr(&buf, tree_hex); +strbuf_addstr(&buf, "\nauthor "); +/* ... */ +``` + +Choose descriptive variable names (`header` for pack headers, not generic +`buf`; use `buf` for the secondary strbuf if you cannot reuse the first). + +## Platform Considerations + +### Windows-specific issues + +On Windows, `unsigned long` is 32 bits even on 64-bit systems. Use `size_t` +for sizes that may exceed 4GB. Be careful with format strings: use `PRIuMAX` +with a cast for `size_t` values. + +## Contributing to Git for Windows + +The primary contribution path for this fork is a PR against +`git-for-windows/git`'s `main` branch. The repository is laid out as a +branch thicket on top of an upstream Git base; see +[Merging-Rebases](#merging-rebases) and +[Analyzing Branch Thickets](#analyzing-branch-thickets) for the +mechanics. + +### Opening a PR + +Push the topic branch to a personal fork on GitHub, then: + +```bash +gh pr create \ + --repo git-for-windows/git \ + --base main \ + --head : \ + --title "" \ + --body-file +``` + +Unlike upstream contributions, the PR body is rendered as Markdown on +GitHub, not sent as email. Use the formatting that aids review: +fenced code blocks, tables, links to workflow runs. + +### When the PR Adjusts the Thicket Itself + +If the PR's purpose is to edit, remove, or replace existing +branch-thicket commits, the natural form is a series of `fixup!` or +`amend!` commits targeting the affected originals. See +[Fixup Commits](#fixup-commits), +[`amend!` Commits](#amend-commits), and +[PRs Composed Entirely of `fixup!` and `amend!` Commits](#prs-composed-entirely-of-fixup-and-amend-commits). +The merging-rebase that produces the next `main` autosquashes these +into the thicket; the PR exists for review of the individual +adjustments. + +### When an Upstream Patch Will Replace a Thicket Commit + +If an upstream patch is in flight (for instance, on `gitgitgadget/git` +in `seen` or `next`) that replaces a downstream thicket commit, an +`amend!` commit whose body is a verbatim copy of the upstream commit +message and whose diff aligns the autosquashed target with the +upstream commit's diff is the canonical pattern. The next +merging-rebase that picks up the upstream commit will recognize the +two as byte-identical and collapse them. +## Contributing to Upstream Git via GitGitGadget + +### Overview + +The upstream Git project accepts contributions via the mailing list +(`git@vger.kernel.org`). [GitGitGadget](https://gitgitgadget.github.io/) +bridges GitHub PRs to the mailing list: you push a branch to your GitHub +fork, open a PR against https://github.com/gitgitgadget/git, and +GitGitGadget formats and sends the patches. + +### Workflow + +1. Push the topic branch to your personal fork on GitHub (the remote + that points at `https://github.com//git`). +2. Open a PR from `:` against `gitgitgadget/git`'s `master`. +3. The PR title becomes the patch series subject; the PR body becomes the + cover letter. Use + `gh pr create --repo gitgitgadget/git --head :`. +4. Use `/submit` as a PR comment to send patches to the mailing list. +5. After review feedback, update the branch, force-push, and `/submit` again. + +### Branch Naming + +Do **not** use an initials prefix (like `ds/` or `js/`). That convention is +used by the Git maintainer when picking up topics, not by contributors. Use +descriptive names like `tests-explicit-bare-repo`. + +### Cover Letter Style + +The PR body is the cover letter. It should be plain text (not Markdown with +headers or bullet formatting), since it will be sent as email. Structure: + +- A brief subject line (the PR title, e.g. "tests: access bare repositories + explicitly") +- Motivation: why is this change needed? +- Summary: what does the series do? What patterns/techniques does it use? +- Scope: is this part of a larger effort? If so, link to the tracking PR. + +Keep it factual and measured. Avoid framing changes in terms of security +when contributing to upstream Git; frame them as robustness, correctness, +or preparation for future defaults. + +### Commit Message Conventions (Upstream Git) + +Upstream Git commit messages follow stricter conventions than the Microsoft +Git fork: + +- **Subject line**: `: ` (lowercase after the colon). + The `` is typically a file name without extension (e.g. `t0001`, + `setup`, `scalar`) or a subsystem name (e.g. `tests`, `refs`). +- **Body**: Flowing English prose, no bullet points. Wrap at 76 columns. +- **ASCII only**: No Unicode characters anywhere in the message. +- **Trailers**: `Signed-off-by` is mandatory. `Assisted-by` for AI. +- The subject line must accurately describe the diff content. If a commit + adds `--git-dir=.` to one invocation, do not title it "wrap bare repo + commands in subshell with `GIT_DIR`". + +### Patch Series with Dependencies + +When contributing a branch thicket (multiple related patch series with +dependencies), submit the foundation series first and note the overall +effort in the cover letter with a link to the tracking PR or `compare` +URL. Submit dependent series after earlier ones land in `seen`. + +Use `git replay --onto ..` to test whether a +sub-branch applies cleanly to a given base (e.g., `upstream/master` or +`upstream/seen`) without touching the working tree. By default (since +the `--ref-action` default changed to `update`), `git replay` updates +named refs in the range directly, producing no stdout output. Use +`--ref-action=print` to get the old behavior of printing `update-ref` +commands to stdout instead. Always verify that `git replay` actually +did something by checking the reflog of the affected branches. + +## Working with Worktrees + +### General Principles + +Use worktrees to work on multiple topics simultaneously without stashing +or switching branches. Keep worktrees as subdirectories of the main +repository and add them to `.git/info/exclude` so they do not show up +as untracked files. + +```bash +git worktree add +echo "" >> .git/info/exclude +``` + +### Rewriting Commits with `--update-refs` + +When rewriting history in a worktree (e.g., fixing a commit message via +`amend!` + autosquash), use `--update-refs` so that other local branches +pointing into the rewritten range are updated automatically: + +```bash +# Create a local branch at the commit to be pushed +git branch + +# Create the amend! commit and autosquash +git commit --allow-empty -F +GIT_SEQUENCE_EDITOR=true GIT_EDITOR=true \ + git rebase -i --autosquash --update-refs + +# Verify: tree should be identical +git diff @{1}.. + +# Force-push the updated branch +git push --force-with-lease +``` + +The `--update-refs` flag is essential: without it, only the checked-out +branch is rewritten and other branches become stale, pointing at +pre-rewrite commits. + +### Verifying Rebase Results + +After any rebase, verify that the tree content is unchanged (unless you +intentionally modified it): + +```bash +git diff @{1} # Should be empty for pure rewording +git range-diff @{1}... # Shows per-commit changes +``` + +## Analyzing Branch Thickets + +When a branch is structured as a sequence of merged sub-branches (a +"branch thicket"), use the merge structure to extract sub-branches: + +```bash +# List the merge commits (sub-branches) +git log --oneline --first-parent ...upstream/master | grep 'Merge branch' + +# Extract commits for a specific sub-branch (second parent of its merge) +git log --oneline ^1..^2 + +# Find what each sub-branch forks from +git log -1 --format='%H %s' ^ +``` + +Use `git replay` to test whether sub-branches can be rebased onto a new +base without conflicts. This replaces speculation about "overlapping files" +with actual evidence: + +```bash +git replay --onto upstream/master .. +``` + +If the range contains merge commits, `git replay` will fail with "replaying +merge commits is not supported yet!" In that case, identify the linear +commit range and replay just those commits. + +## Resources + +- [Git for Windows](https://gitforwindows.org/) +- [Git Internals](https://git-scm.com/book/en/v2/Git-Internals-Plumbing-and-Porcelain) +- [GitGitGadget](https://gitgitgadget.github.io/) - Bridge GitHub PRs to + the Git mailing list +- [Git Mailing List Archive](https://lore.kernel.org/git/) - Searchable + archive of all upstream discussion diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000000000..7de4f99bf71ec4 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,116 @@ +# Architecture of Git for Windows + +Git for Windows is a complex project. + +## What _is_ Git for Windows? + +### A fork of `git/git` + +First and foremost, it is a friendly fork of [`git/git`](https://github.com/git/git), aiming to improve Git's Windows support. The [`git-for-windows/git`](https://github.com/git-for-windows/git) repository contains dozens of topics on top of `git/git`, some awaiting to be "upstreamed" (i.e. to be contributed to `git/git`), some still being stabilized, and a few topics are specific to the Git for Windows project and are not intended to be integrated into `git/git` at all. + +### Enhancing and maintaining Git's support for Windows + +On the source code side, Git's Windows support is made a bit more tricky than strictly necessary by the fact that Git does not have any platform abstraction layer (unlike other version control systems, such as Subversion). It relies on the presence of POSIX features such as the `hstrerror()` function, and on platforms lacking that functionality, Git provides shims. That leads to some challenges e.g. with the `stat()` function which is very slow on Windows because it has to collect much more metadata than what e.g. the very quick `GetFileAttributesExW()` Win32 API function provides, even when Git calls `stat()` merely to test for the presence of a file (for which all that gathered metadata is totally irrelevant). + +### Providing more than just source code + +In contrast to the Git project, Git for Windows not only publishes tagged source code versions, but full builds of Git. In fact, Git for Windows' primary purpose, as far as most users are concerned, is to provide a convenient installer that end-users can run to have Git on their computer, without ever having to check out `git-for-windows/git` let alone build it. In essence, Git for Windows has to maintain a separate project altogether in addition to the fork of `git/git`, just to build these release artifacts: [`git-for-windows/build-extra`](https://github.com/git-for-windows/build-extra). This repository also contains the definition for a couple of other release artifacts published by Git for Windows, e.g. the "portable" edition of Git for Windows which is a self-extracting 7-Zip archive that does not need to be installed. + +### A software distribution, really + +Another aspect that contributes to the complexity of Git for Windows is that it is not just building `git.exe` and distributes that. Due to its heritage within the Linux project, Git takes certain things for granted, such as the presence of a Unix shell, or for that matter, a package management system from which dependencies can be fetched and updated independently of Git itself. Things that are distinctly not present in most Windows setups. To accommodate for that, Git for Windows originally relied on the MSys project, a minimal fork of Cygwin providing a Unix shell ("Bash"), a Perl interpreter and similar Unix-like tools, and on the MINGW project, a project to build libraries and executables using a GNU C Compiler that relies only on Win32 API functions. As of Git for Windows v2.x, the project has switched away from [MSys](https://sourceforge.net/projects/mingw/files/MSYS/)/[MinGW](https://osdn.net/projects/mingw/) (due to less-than-active maintenance) to [the MSYS2 project](https://msys2.org). That switch brought along the benefit of a robust package management system based on [Pacman](https://archlinux.org/pacman/) (hailing from Arch Linux). To support Windows users, who are in general unfamiliar with Linux-like package management and the need to update installed packages frequently, Git for Windows bundles a subset of its own fork of MSYS2. To put things in perspective: Git for Windows bundles files from ~170 packages, one of which contains Git, and another one contains Git's help files. In that respect, Git for Windows acts like a distribution more than like a mere single software application. + +Most of MSYS2's packages that are bundled in Git for Windows are consumed directly from MSYS2. Others need forks that are maintained by Git for Windows project, to support Git for Windows better. These forks live in the [`git-for-windows/MSYS2-packages`](https://github.com/git-for-windows/MSYS2-packages) and [`git-for-windows/MINGW-packages`](https://github.com/git-for-windows/MINGW-packages) repositories. There are several reasons justifying these forks. For example, the Git for Windows' flavor of the MSYS2 runtime behaves like Git's test suite expects it while MSYS2's flavor does not. Another example: The Bash executable bundled in Git for Windows is code-signed with the same certificate as `git.exe` to help anti-malware programs get out of the users' way. That is why Git for Windows maintains its own `bash` Pacman package. And since MSYS2 dropped 32-bit support already, Git for Windows has to update the 32-bit Pacman packages itself, which is done in the git-for-windows/MSYS2-packages repository. (Side note: the 32-bit issue is a bit more complicated, actually: MSYS2 _still_ builds _MINGW_ packages targeting i686 processors, but no longer any _MSYS_ packages for said processor architecture, and Git for Windows does not keep all of the 32-bit MSYS packages up to date but instead judiciously decides which packages are vital enough as far as Git is concerned to justify the maintenance cost.) + +### Supporting third-party applications that use Git's functionality + +Since the infrastructure required by Git is non-trivial the installer (or for that matter, the Portable Git) is not exactly light-weight: As of January 2023, both artifacts are over fifty megabytes. This is a problem for third-party applications wishing to bundle a version of Git for Windows, which is often advisable given that applications may depend on features that have been introduced only in recent Git versions and therefore relying on an installed Git for Windows could break things. To help with that, the Git for Windows project also provides MinGit as a release artifact, a zip file that is much smaller than the full installer and that contains only the parts of Git for Windows relevant for third-party applications. It lacks Git GUI, for example, as well as the terminal program MinTTY, or for that matter, the documentation. + +### Supporting `git/git`'s GitHub workflows + +The Git for Windows project is also responsible for keeping the Windows part of `git/git`'s automated builds up and running. On Windows, there is no canonical and easy way to get a build environment necessary to build Git and run its test suite, therefore this is a non-trivial task that comes with its own maintenance cost. Git for Windows provides two GitHub Actions to help with that: [`git-for-windows/setup-git-for-windows-sdk`](https://github.com/git-for-windows/setup-git-for-windows-sdk) to set up a tiny subset of Git for Windows' full SDK (which would require about 500MB to be cloned, as opposed to the ~75MB of that subset) and [`git-for-windows/get-azure-pipelines-artifact`](https://github.com/git-for-windows/get-azure-pipelines-artifact) e.g. to download some regularly pre-built artifacts (for example, when `git/git`'s automated tests ran on an Ubuntu version that did not provide an up to date [Coccinelle](https://coccinelle.gitlabpages.inria.fr/website/) package, this GitHub Action was used to download a pre-built version of that Debian package). + +## Maintaining Git for Windows' components + +Git for Windows uses a combination of [a GitHub App called GitForWindowsHelper](https://github.com/git-for-windows/gfw-helper-github-app) (to listen for so-called [slash commands](https://github.com/git-for-windows/gfw-helper-github-app#slash-commands)) combined with workflows in [the `git-for-windows-automation` repository](https://github.com/git-for-windows/git-for-windows-automation/) (for computationally heavy tasks) to support Git for Windows' repetitive tasks. + +This heavy automation serves two purposes: + +1. Document the knowledge about "how things are done" in the Git for Windows project. +2. Make Git for Windows' maintenance less tedious by off-loading as many tasks onto machines as possible. + +One neat trick of some `git-for-windows-automation` workflows is that they "mirror back" check runs to the targeted PRs in another repository. This essentially allows versioning the source code independently of the workflow definition. + +Here is a diagram showing how the bits and pieces fit together. + +```mermaid +graph LR + A[`monitor-components`] --> |opens| B + B{issues labeled
`component-update`} --> |/open pr| C + C((GitForWindowsHelper)) --> |triggers| D + D[`open-pr`] --> |opens| E + E{PR in
MINGW-packages
MSYS2-packages
build-extra} --> |closes| B + E --> |/deploy| F + F((GitForWindowsHelper)) --> |triggers| G + G[`build-and-deploy`] --> |deploys to| H + H{Pacman repository} + C --> |backed by| I + F --> |backed by| I + I[[Azure Function]] + D --> |running in| J + G --> | running in| J + J[[git-for-windows-automation]] + K[[git-sdk-32
git-sdk-64
git-sdk-arm64]] --> |syncing from| H + B --> |/add release note| L + L[`add-release-note`] +``` + +For the curious mind, here are [detailed instructions how the Azure Function backing the GitForWindowsHelper GitHub App was set up](https://github.com/git-for-windows/gfw-helper-github-app#how-this-github-app-was-set-up). + +### The `monitor-components` workflow + +When new versions of components that Git for Windows builds become available, new Pacman packages have to be built. To this end, [the `monitor-components` workflow](https://github.com/git-for-windows/git/blob/main/.github/workflows/monitor-components.yml) monitors a couple of RSS feeds and opens new tickets labeled `component-update` for such new versions. + +### Opening Pull Requests to update Git for Windows' components + +After determining that such a ticket indeed indicates the need for a new Pacman package build, a Git for Windows maintainer issues the `/open pr` command via an issue comment ([example](https://github.com/git-for-windows/git/issues/4281#issuecomment-1426859787)), which gets picked up by the GitForWindowsHelper GitHub App, which in turn triggers [the `open-pr` workflow](https://github.com/git-for-windows/git-for-windows-automation/blob/main/.github/workflows/open-pr.yml) in the `git-for-windows-automation` repository. + +### Deploying the Pacman packages + +This will open a Pull Request in one of Git for Windows' repositories, and once the PR build passes, a Git for Windows maintainer issues the `/deploy` command ([example](https://github.com/git-for-windows/MINGW-packages/pull/69#issuecomment-1427591890)), which gets picked up by the GitForWindowsHelper GitHub App, which triggers [the `build-and-deploy` workflow](https://github.com/git-for-windows/git-for-windows-automation/blob/main/.github/workflows/build-and-deploy.yml). + +### Adding release notes + +Finally, once the packages have been built and deployed to the Pacman repository (which is hosted in Azure Blob Storage), a Git for Windows maintainer will merge the PR(s), which in turn will close the ticket, and the maintainer then issues an `/add release note` command ([example](https://github.com/git-for-windows/MINGW-packages/pull/69#issuecomment-1427782230)), which again gets picked up by the GitForWindowsHelper GitHub App that triggers [the `add-release-note` workflow](https://github.com/git-for-windows/build-extra/blob/main/.github/workflows/add-release-note.yml) that creates and pushes a new commit to the `ReleaseNotes.md` file in `build-extra` ([example](https://github.com/git-for-windows/build-extra/commit/b39c148ff8dc0e987afdb677d17c46a8e99fd0ef)). + +## Releasing official Git for Windows versions + +A relatively infrequent part of Git for Windows' maintainers' duties, if the most rewarding part, is the task of releasing new versions of Git for Windows. + +Most commonly, this is done in response to the "upstream" Git project releasing a new version. When that happens, a Git for Windows maintainer runs [the helper script](https://github.com/git-for-windows/build-extra/blob/main/shears.sh) to perform a "merging rebase" (i.e. a rebase that starts with a fake-merge of the previous tip commit, to maintain both a clean set of commits as well as a [fast-forwarding](https://git-scm.com/docs/git-merge#Documentation/git-merge.txt---ff-only) commit history). + +Once that is done, the maintainer will open a Pull Request to benefit from the automated builds and tests ([example](https://github.com/git-for-windows/git/pull/4160)) as well as from reviews of the [`range-diff`](https://git-scm.com/docs/git-range-diff) relative to the current `main` branch. + +Once everything looks good, the maintainer will issue the `/git-artifacts` command ([example](https://github.com/git-for-windows/git/pull/4160#issuecomment-1346801735)). This will trigger an automated workflow that builds all of the release artifacts: installers, Portable Git, MinGit, `.tar.xz` archive and a NuGet package. Apart from the NuGet package, two sets of artifacts are built: targeting 32-bit ("x86") and 64-bit ("amd64"). + +Once these artifacts are built, the maintainer will download the installer and run [the "pre-flight checklist"](https://github.com/git-for-windows/build-extra/blob/main/installer/checklist.txt). + +If everything looks good, a `/release` command will be issued, which triggers yet another workflow that will download the just-built-and-verified release artifacts, publish them as a new GitHub release, publish the NuGet packages, deploy the Pacman packages to the Pacman repository, send out an announcement mail, and update the respective repositories including [Git for Windows' website](https://gitforwindows.org/). + +As mentioned [before](#architecture-of-git-for-windows), the `/git-artifacts` and `/release` commands are picked up by the GitForWindowsHelper GitHub App which subsequently triggers the respective workflows in the `git-for-windows-automation` repository. Here is a diagram: + +```mermaid +graph LR + A{Pull Request
updating to
new Git version} --> |/git-artifacts| B + B((GitForWindowsHelper)) --> |triggers| C + C[`tag-git`] --> |upon successful build
triggers| D + D((GitForWindowsHelper)) --> |triggers| E + E[`git-artifacts`] + E --> |maintainer verifies artifacts| E + A --> |upon verified `git-artifacts`
/release| F + F[`release-git`] + C --> |running in| J + E --> | running in| J + F --> | running in| J + J[[git-for-windows-automation]] +``` \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index e58917c50a96dc..4daef7e3ce9196 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,9 +1,9 @@ -# Git Code of Conduct +# Git for Windows Code of Conduct This code of conduct outlines our expectations for participants within -the Git community, as well as steps for reporting unacceptable behavior. -We are committed to providing a welcoming and inspiring community for -all and expect our code of conduct to be honored. Anyone who violates +the **Git for Windows** community, as well as steps for reporting unacceptable +behavior. We are committed to providing a welcoming and inspiring community +for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community. ## Our Pledge @@ -12,8 +12,8 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. @@ -28,17 +28,17 @@ community include: * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community +* Focusing on what is best not just for us as individuals, but for the overall + community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind +* The use of sexualized language or imagery, and sexual attention or advances of + any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting @@ -58,20 +58,14 @@ decisions when appropriate. This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, +Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -git@sfconservancy.org, or individually: - - - Ævar Arnfjörð Bjarmason - - Christian Couder - - Junio C Hamano - - Taylor Blau +reported by contacting the Git for Windows maintainer. All complaints will be reviewed and investigated promptly and fairly. @@ -94,15 +88,15 @@ behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: A violation through a single incident or series of +actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. +like social media. Violating these terms may lead to a temporary or permanent +ban. ### 3. Temporary Ban @@ -118,27 +112,27 @@ Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an +standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within -the community. +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. For answers to common questions about this code of conduct, see the FAQ at -[https://www.contributor-covenant.org/faq][FAQ]. Translations are available -at [https://www.contributor-covenant.org/translations][translations]. +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. [homepage]: https://www.contributor-covenant.org -[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html [Mozilla CoC]: https://github.com/mozilla/diversity [FAQ]: https://www.contributor-covenant.org/faq [translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000000..48ff9029374df3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,417 @@ +How to Contribute to Git for Windows +==================================== + +Git was originally designed for Unix systems and still today, all the build tools for the Git +codebase assume you have standard Unix tools available in your path. If you have an open-source +mindset and want to start contributing to Git, but primarily use a Windows machine, then you may +have trouble getting started. This guide is for you. + +Get the Source +-------------- + +Clone the [GitForWindows repository on GitHub](https://github.com/git-for-windows/git). +It is helpful to create your own fork for storing your development branches. + +Windows uses different line endings than Unix systems. See +[this GitHub article on working with line endings](https://help.github.com/articles/dealing-with-line-endings/#refreshing-a-repository-after-changing-line-endings) +if you have trouble with line endings. + +Build the Source +---------------- + +First, download and install the latest [Git for Windows SDK (64-bit)](https://github.com/git-for-windows/build-extra/releases/latest). +When complete, you can run the Git SDK, which creates a new Git Bash terminal window with +the additional development commands, such as `make`. + + As of time of writing, the SDK uses a different credential manager, so you may still want to use normal Git + Bash for interacting with your remotes. Alternatively, use SSH rather than HTTPS and + avoid credential manager problems. + +You should now be ready to type `make` from the root of your `git` source directory. +Here are some helpful variations: + +* `make -j[N] DEVELOPER=1`: Compile new sources using up to N concurrent processes. + The `DEVELOPER` flag turns on all warnings; code failing these warnings will not be + accepted upstream ("upstream" = "the core Git project"). +* `make clean`: Delete all compiled files. + +When running `make`, you can use `-j$(nproc)` to automatically use the number of processors +on your machine as the number of concurrent build processes. + +You can go deeper on the Windows-specific build process by reading the +[technical overview](https://gitforwindows.org/technical-overview) or the +[guide to compiling Git with Visual Studio](https://gitforwindows.org/compiling-git-with-visual-studio). + +## Building `git` on Windows with Visual Studio + +The typical approach to building `git` is to use the standard `Makefile` with GCC, as +above. Developers working in a Windows environment may want to instead build with the +[Microsoft Visual C++ compiler and libraries toolset (MSVC)](https://blogs.msdn.microsoft.com/vcblog/2017/03/07/msvc-the-best-choice-for-windows/). +There are a few benefits to using MSVC over GCC during your development, including creating +symbols for debugging and [performance tracing](https://github.com/Microsoft/perfview#perfview-overview). + +There are two ways to build Git for Windows using MSVC. Each have their own merits. + +### Using SDK Command Line + +Use one of the following commands from the SDK Bash window to build Git for Windows: + +``` + make MSVC=1 -j12 + make MSVC=1 DEBUG=1 -j12 +``` + +The first form produces release-mode binaries; the second produces debug-mode binaries. +Both forms produce PDB files and can be debugged. However, the first is best for perf +tracing and the second is best for single-stepping. + +You can then open Visual Studio and select File -> Open -> Project/Solution and select +the compiled `git.exe` file. This creates a basic solution and you can use the debugging +and performance tracing tools in Visual Studio to monitor a Git process. Use the Debug +Properties page to set the working directory and command line arguments. + +Be sure to clean up before switching back to GCC (or to switch between debug and +release MSVC builds): + +``` + make MSVC=1 -j12 clean + make MSVC=1 DEBUG=1 -j12 clean +``` + +### Using the IDE + +If you prefer working in Visual Studio with a solution full of projects, then you can use +CMake, either by letting Visual Studio configure it automatically (simply open Git's +top-level directory via `File>Open>Folder...`) or by (downloading and) running +[CMake](https://cmake.org) manually. + +What to Change? +--------------- + +Many new contributors ask: What should I start working on? + +One way to win big with the open-source community is to look at the +[issues page](https://github.com/git-for-windows/git/issues) and see if there are any issues that +you can fix quickly, or if anything catches your eye. + +You can also look at [the unofficial Chromium issues page](https://crbug.com/git) for +multi-platform issues. You can look at recent user questions on +[the Git mailing list](https://public-inbox.org/git). + +Or you can "scratch your own itch", i.e. address an issue you have with Git. The team at Microsoft where the Git for Windows maintainer works, for example, is focused almost entirely on [improving performance](https://blogs.msdn.microsoft.com/devops/2018/01/11/microsofts-performance-contributions-to-git-in-2017/). +We approach our work by finding something that is slow and try to speed it up. We start our +investigation by reliably reproducing the slow behavior, then running that example using +the MSVC build and tracing the results in PerfView. + +You could also think of something you wish Git could do, and make it do that thing! The +only concern I would have with this approach is whether or not that feature is something +the community also wants. If this excites you though, go for it! Don't be afraid to +[get involved in the mailing list](http://vger.kernel.org/vger-lists.html#git) early for +feedback on the idea. + +Test Your Changes +----------------- + +After you make your changes, it is important that you test your changes. Manual testing is +important, but checking and extending the existing test suite is even more important. You +want to run the functional tests to see if you broke something else during your change, and +you want to extend the functional tests to be sure no one breaks your feature in the future. + +### Functional Tests + +Navigate to the `t/` directory and type `make` to run all tests or use `prove` as +[described on this Git for Windows page](https://gitforwindows.org/building-git): + +``` +prove -j12 --state=failed,save ./t[0-9]*.sh +``` + +You can also run each test directly by running the corresponding shell script with a name +like `tNNNN-descriptor.sh`. + +If you are adding new functionality, you may need to create unit tests by creating +helper commands that test a very limited action. These commands are stored in `t/helpers`. +When adding a helper, be sure to add a line to `t/Makefile` and to the `.gitignore` for the +binary file you add. The Git community prefers functional tests using the full `git` +executable, so try to exercise your new code using `git` commands before creating a test +helper. + +To find out why a test failed, repeat the test with the `-x -v -d -i` options and then +navigate to the appropriate "trash" directory to see the data shape that was used for the +test failed step. + +Read [`t/README`](t/README) for more details. + +### Performance Tests + +If you are working on improving performance, you will need to be acquainted with the +performance tests in `t/perf`. There are not too many performance tests yet, but adding one +as your first commit in a patch series helps to communicate the boost your change provides. + +To check the change in performance across multiple versions of `git`, you can use the +`t/perf/run` script. For example, to compare the performance of `git rev-list` across the +`core/master` and `core/next` branches compared to a `topic` branch, you can run + +``` +cd t/perf +./run core/master core/next topic -- p0001-rev-list.sh +``` + +You can also set certain environment variables to help test the performance on different +repositories or with more repetitions. The full list is available in +[the `t/perf/README` file](t/perf/README), +but here are a few important ones: + +``` +GIT_PERF_REPO=/path/to/repo +GIT_PERF_LARGE_REPO=/path/to/large/repo +GIT_PERF_REPEAT_COUNT=10 +``` + +When running the performance tests on Linux, you may see a message "Can't locate JSON.pm in +@INC" and that means you need to run `sudo cpanm install JSON` to get the JSON perl package. + +For running performance tests, it can be helpful to set up a few repositories with strange +data shapes, such as: + +**Many objects:** Clone repos such as [Kotlin](https://github.com/jetbrains/kotlin), [Linux](https://github.com/torvalds/linux), or [Android](https://source.android.com/setup/downloading). + +**Many pack-files:** You can split a fresh clone into multiple pack-files of size at most +16MB by running `git repack -adfF --max-pack-size=16m`. See the +[`git repack` documentation](https://git-scm.com/docs/git-repack) for more information. +You can count the number of pack-files using `ls .git/objects/pack/*.pack | wc -l`. + +**Many loose objects:** If you already split your repository into multiple pack-files, then +you can pick one to split into loose objects using `cat .git/objects/pack/[id].pack | git unpack-objects`; +delete the `[id].pack` and `[id].idx` files after this. You can count the number of loose +bjects using `ls .git/objects/??/* | wc -l`. + +**Deep history:** Usually large repositories also have deep histories, but you can use the +[test-many-commits-1m repo](https://github.com/cirosantilli/test-many-commits-1m/) to +target deep histories without the overhead of many objects. One issue with this repository: +there are no merge commits, so you will need to use a different repository to test a "wide" +commit history. + +**Large Index:** You can generate a large index and repo by using the scripts in +`t/perf/repos`. There are two scripts. `many-files.sh` which will generate a repo with +same tree and blobs but different paths. Using `many-files.sh -d 5 -w 10 -f 9` will create +a repo with ~1 million entries in the index. `inflate-repo.sh` will use an existing repo +and copy the current work tree until it is a specified size. + +Test Your Changes on Linux +-------------------------- + +It can be important to work directly on the [core Git codebase](https://github.com/git/git), +such as a recent commit into the `master` or `next` branch that has not been incorporated +into Git for Windows. Also, it can help to run functional and performance tests on your +code in Linux before submitting patches to the mailing list, which focuses on many platforms. +The differences between Windows and Linux are usually enough to catch most cross-platform +issues. + +### Using the Windows Subsystem for Linux + +The [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10) +allows you to [install Ubuntu Linux as an app](https://www.microsoft.com/en-us/store/p/ubuntu/9nblggh4msv6) +that can run Linux executables on top of the Windows kernel. Internally, +Linux syscalls are interpreted by the WSL, everything else is plain Ubuntu. + +First, open WSL (either type "Bash" in Cortana, or execute "bash.exe" in a CMD window). +Then install the prerequisites, and `git` for the initial clone: + +``` +sudo apt-get update +sudo apt-get install git gcc make libssl-dev libcurl4-openssl-dev \ + libexpat-dev tcl tk gettext git-email zlib1g-dev +``` + +Then, clone and build: + +``` +git clone https://github.com/git-for-windows/git +cd git +git remote add -f upstream https://github.com/git/git +make +``` + +Be sure to clone into `/home/[user]/` and not into any folder under `/mnt/?/` or your build +will fail due to colons in file names. + +### Using a Linux Virtual Machine with Hyper-V + +If you prefer, you can use a virtual machine (VM) to run Linux and test your changes in the +full environment. The test suite runs a lot faster on Linux than on Windows or with the WSL. +You can connect to the VM using an SSH terminal like +[PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/). + +The following instructions are for using Hyper-V, which is available in some versions of Windows. +There are many virtual machine alternatives available, if you do not have such a version installed. + +* [Download an Ubuntu Server ISO](https://www.ubuntu.com/download/server). +* Open [Hyper-V Manager](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/quick-start/enable-hyper-v). +* [Set up a virtual switch](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/quick-start/connect-to-network) + so your VM can reach the network. +* Select "Quick Create", name your machine, select the ISO as installation source, and un-check + "This virtual machine will run Windows." +* Go through the Ubuntu install process, being sure to select to install OpenSSH Server. +* When install is complete, log in and check the SSH server status with `sudo service ssh status`. + * If the service is not found, install with `sudo apt-get install openssh-server`. + * If the service is not running, then use `sudo service ssh start`. +* Use `shutdown -h now` to shutdown the VM, go to the Hyper-V settings for the VM, expand Network Adapter + to select "Advanced Features", and set the MAC address to be static (this can save your VM from losing + network if shut down incorrectly). +* Provide as many cores to your VM as you can (for parallel builds). +* Restart your VM, but do not connect. +* Use `ssh` in Git Bash, download [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/), or use your favorite SSH client to connect to the VM through SSH. + +In order to build and use `git`, you will need the following libraries via `apt-get`: + +``` +sudo apt-get update +sudo apt-get install git gcc make libssl-dev libcurl4-openssl-dev \ + libexpat-dev tcl tk gettext git-email zlib1g-dev +``` + +To get your code from your Windows machine to the Linux VM, it is easiest to push the branch to your fork of Git and clone your fork in the Linux VM. + +Don't forget to set your `git` config with your preferred name, email, and editor. + +Polish Your Commits +------------------- + +Before submitting your patch, be sure to read the [coding guidelines](https://github.com/git/git/blob/master/Documentation/CodingGuidelines) +and check your code to match as best you can. This can be a lot of effort, but it saves +time during review to avoid style issues. + +The other possibly major difference between the mailing list submissions and GitHub PR workflows +is that each commit will be reviewed independently. Even if you are submitting a +patch series with multiple commits, each commit must stand on it's own and be reviewable +by itself. Make sure the commit message clearly explain the why of the commit not the how. +Describe what is wrong with the current code and how your changes have made the code better. + +When preparing your patch, it is important to put yourself in the shoes of the Git community. +Accepting a patch requires more justification than approving a pull request from someone on +your team. The community has a stable product and is responsible for keeping it stable. If +you introduce a bug, then they cannot count on you being around to fix it. When you decided +to start work on a new feature, they were not part of the design discussion and may not +even believe the feature is worth introducing. + +Questions to answer in your patch message (and commit messages) may include: +* Why is this patch necessary? +* How does the current behavior cause pain for users? +* What kinds of repositories are necessary for noticing a difference? +* What design options did you consider before writing this version? Do you have links to + code for those alternate designs? +* Is this a performance fix? Provide clear performance numbers for various well-known repos. + +Here are some other tips that we use when cleaning up our commits: + +* Commit messages should be wrapped at 76 columns per line (or less; 72 is also a + common choice). +* Make sure the commits are signed off using `git commit (-s|--signoff)`. See + [SubmittingPatches](https://github.com/git/git/blob/v2.8.1/Documentation/SubmittingPatches#L234-L286) + for more details about what this sign-off means. +* Check for whitespace errors using `git diff --check [base]...HEAD` or `git log --check`. +* Run `git rebase --whitespace=fix` to correct upstream issues with whitespace. +* Become familiar with interactive rebase (`git rebase -i`) because you will be reordering, + squashing, and editing commits as your patch or series of patches is reviewed. +* Make sure any shell scripts that you add have the executable bit set on them. This is + usually for test files that you add in the `/t` directory. You can use + `git add --chmod=+x [file]` to update it. You can test whether a file is marked as executable + using `git ls-files --stage \*.sh`; the first number is 100755 for executable files. +* Your commit titles should match the "area: change description" format. Rules of thumb: + * Choose ": " prefix appropriately. + * Keep the description short and to the point. + * The word that follows the ": " prefix is not capitalized. + * Do not include a full-stop at the end of the title. + * Read a few commit messages -- using `git log origin/master`, for instance -- to + become acquainted with the preferred commit message style. +* Build source using `make DEVELOPER=1` for extra-strict compiler warnings. + +Submit Your Patch +----------------- + +Git for Windows [accepts pull requests on GitHub](https://github.com/git-for-windows/git/pulls), but +these are reserved for Windows-specific improvements. For core Git, submissions are accepted on +[the Git mailing list](https://public-inbox.org/git). + +### Configure Git to Send Emails + +There are a bunch of options for configuring the `git send-email` command. These options can +be found in the documentation for +[`git config`](https://git-scm.com/docs/git-config) and +[`git send-email`](https://git-scm.com/docs/git-send-email). + +``` +git config --global sendemail.smtpserver +git config --global sendemail.smtpserverport 587 +git config --global sendemail.smtpencryption tls +git config --global sendemail.smtpuser +``` + +To avoid storing your password in the config file, store it in the Git credential manager: + +``` +$ git credential fill +protocol=smtp +host= +username= +password=password +``` + +Before submitting a patch, read the [Git documentation on submitting patches](https://github.com/git/git/blob/master/Documentation/SubmittingPatches). + +To construct a patch set, use the `git format-patch` command. There are three important options: + +* `--cover-letter`: If specified, create a `[v#-]0000-cover-letter.patch` file that can be + edited to describe the patch as a whole. If you previously added a branch description using + `git branch --edit-description`, you will end up with a 0/N mail with that description and + a nice overall diffstat. +* `--in-reply-to=[Message-ID]`: This will mark your cover letter as replying to the given + message (which should correspond to your previous iteration). To determine the correct Message-ID, + find the message you are replying to on [public-inbox.org/git](https://public-inbox.org/git) and take + the ID from between the angle brackets. + +* `--subject-prefix=[prefix]`: This defaults to [PATCH]. For subsequent iterations, you will want to + override it like `--subject-prefix="[PATCH v2]"`. You can also use the `-v` option to have it + automatically generate the version number in the patches. + +If you have multiple commits and use the `--cover-letter` option be sure to open the +`0000-cover-letter.patch` file to update the subject and add some details about the overall purpose +of the patch series. + +### Examples + +To generate a single commit patch file: +``` +git format-patch -s -o [dir] -1 +``` +To generate four patch files from the last three commits with a cover letter: +``` +git format-patch --cover-letter -s -o [dir] HEAD~4 +``` +To generate version 3 with four patch files from the last four commits with a cover letter: +``` +git format-patch --cover-letter -s -o [dir] -v 3 HEAD~4 +``` + +### Submit the Patch + +Run [`git send-email`](https://git-scm.com/docs/git-send-email), starting with a test email: + +``` +git send-email --to=yourself@address.com [dir with patches]/*.patch +``` + +After checking the receipt of your test email, you can send to the list and to any +potentially interested reviewers. + +``` +git send-email --to=git@vger.kernel.org --cc= --cc= [dir with patches]/*.patch +``` + +To submit a nth version patch (say version 3): + +``` +git send-email --to=git@vger.kernel.org --cc= --cc= \ + --in-reply-to= [dir with patches]/*.patch +``` diff --git a/Documentation/config.adoc b/Documentation/config.adoc index a80e7db46d9697..672af62f5d10a0 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -521,6 +521,8 @@ include::config/safe.adoc[] include::config/sendemail.adoc[] +include::config/sendpack.adoc[] + include::config/sequencer.adoc[] include::config/showbranch.adoc[] @@ -539,6 +541,8 @@ include::config/status.adoc[] include::config/submodule.adoc[] +include::config/survey.adoc[] + include::config/tag.adoc[] include::config/tar.adoc[] @@ -561,4 +565,6 @@ include::config/versionsort.adoc[] include::config/web.adoc[] +include::config/windows.adoc[] + include::config/worktree.adoc[] diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index 257db58918179a..28fb0e4a18f9f3 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -64,6 +64,9 @@ all advice messages. set their identity configuration. mergeConflict:: Shown when various commands stop because of conflicts. + nameTooLong:: + Advice shown if a filepath operation is attempted where the + path was too long. nestedTag:: Shown when a user attempts to recursively tag a tag object. pushAlreadyExists:: @@ -166,4 +169,8 @@ all advice messages. Shown when the user tries to create a worktree from an invalid reference, to tell the user how to create a new unborn branch instead. + + useCoreFSMonitorConfig:: + Advice shown if the deprecated 'core.useBuiltinFSMonitor' config + setting is in use. -- diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index a0ebf03e2eb050..cac7438e7de505 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -721,6 +721,19 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. @@ -788,3 +801,9 @@ core.maxTreeDepth:: to allow Git to abort cleanly, and should not generally need to be adjusted. When Git is compiled with MSVC, the default is 512. Otherwise, the default is 2048. + +core.WSLCompat:: + Tells Git whether to enable wsl compatibility mode. + The default value is false. When set to true, Git will set the mode + bits of the file in the way of wsl, so that the executable flag of + files can be set or read correctly. diff --git a/Documentation/config/http.adoc b/Documentation/config/http.adoc index 792a71b41350d4..3c97f5ec527480 100644 --- a/Documentation/config/http.adoc +++ b/Documentation/config/http.adoc @@ -242,13 +242,20 @@ http.sslKeyType:: See also libcurl `CURLOPT_SSLKEYTYPE`. Can be overridden by the `GIT_SSL_KEY_TYPE` environment variable. +http.allowNTLMAuth:: + Whether or not to allow NTLM authentication. While very convenient to set + up, and therefore still used in many on-prem scenarios, NTLM is a weak + authentication method and therefore deprecated. Defaults to "false". + http.schannelCheckRevoke:: Used to enforce or disable certificate revocation checks in cURL - when http.sslBackend is set to "schannel". Defaults to `true` if - unset. Only necessary to disable this if Git consistently errors - and the message is about checking the revocation status of a - certificate. This option is ignored if cURL lacks support for - setting the relevant SSL option at runtime. + when http.sslBackend is set to "schannel" via "true" and "false", + respectively. Another accepted value is "best-effort" (the default) + in which case revocation checks are performed, but errors due to + revocation list distribution points that are offline are silently + ignored, as well as errors due to certificates missing revocation + list distribution points. This option is ignored if cURL lacks + support for setting the relevant SSL option at runtime. http.schannelUseSSLCAInfo:: As of cURL v7.60.0, the Secure Channel backend can use the @@ -258,6 +265,11 @@ http.schannelUseSSLCAInfo:: when the `schannel` backend was configured via `http.sslBackend`, unless `http.schannelUseSSLCAInfo` overrides this behavior. +http.sslAutoClientCert:: + As of cURL v7.77.0, the Secure Channel backend won't automatically + send client certificates from the Windows Certificate Store anymore. + To opt in to the old behavior, http.sslAutoClientCert can be set. + http.pinnedPubkey:: Public key of the https service. It may either be the filename of a PEM or DER encoded public key file or a string starting with diff --git a/Documentation/config/sendpack.adoc b/Documentation/config/sendpack.adoc new file mode 100644 index 00000000000000..e306f657fba7dd --- /dev/null +++ b/Documentation/config/sendpack.adoc @@ -0,0 +1,5 @@ +sendpack.sideband:: + Allows to disable the side-band-64k capability for send-pack even + when it is advertised by the server. Makes it possible to work + around a limitation in the git for windows implementation together + with the dump git protocol. Defaults to true. diff --git a/Documentation/config/survey.adoc b/Documentation/config/survey.adoc new file mode 100644 index 00000000000000..9e594a2092f225 --- /dev/null +++ b/Documentation/config/survey.adoc @@ -0,0 +1,14 @@ +survey.*:: + These variables adjust the default behavior of the `git survey` + command. The intention is that this command could be run in the + background with these options. ++ +-- + verbose:: + This boolean value implies the `--[no-]verbose` option. + progress:: + This boolean value implies the `--[no-]progress` option. + top:: + This integer value implies `--top=`, specifying the + number of entries in the detail tables. +-- diff --git a/Documentation/config/windows.adoc b/Documentation/config/windows.adoc new file mode 100644 index 00000000000000..fdaaf1c65504f3 --- /dev/null +++ b/Documentation/config/windows.adoc @@ -0,0 +1,4 @@ +windows.appendAtomically:: + By default, append atomic API is used on windows. But it works only with + local disk files, if you're working on a network file system, you should + set it false to turn it off. diff --git a/Documentation/git-reset.adoc b/Documentation/git-reset.adoc index 5023b5069972ca..933e2fac7dd662 100644 --- a/Documentation/git-reset.adoc +++ b/Documentation/git-reset.adoc @@ -12,6 +12,7 @@ git reset [--soft | --mixed [-N] | --hard | --merge | --keep] [-q] [] git reset [-q] [] [--] ... git reset [-q] [--pathspec-from-file= [--pathspec-file-nul]] [] git reset (--patch | -p) [] [--] [...] +DEPRECATED: git reset [-q] [--stdin [-z]] [] DESCRIPTION ----------- @@ -139,6 +140,16 @@ include::diff-context-options.adoc[] + For more details, see the 'pathspec' entry in linkgit:gitglossary[7]. +`--stdin`:: + DEPRECATED (use `--pathspec-from-file=-` instead): Instead of taking + list of paths from the command line, read list of paths from the + standard input. Paths are separated by LF (i.e. one path per line) by + default. + +`-z`:: + DEPRECATED (use `--pathspec-file-nul` instead): Only meaningful with + `--stdin`; paths are separated with NUL character instead of LF. + EXAMPLES -------- diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc new file mode 100644 index 00000000000000..44f3a0568b7697 --- /dev/null +++ b/Documentation/git-survey.adoc @@ -0,0 +1,83 @@ +git-survey(1) +============= + +NAME +---- +git-survey - EXPERIMENTAL: Measure various repository dimensions of scale + +SYNOPSIS +-------- +[verse] +(EXPERIMENTAL!) 'git survey' + +DESCRIPTION +----------- + +Survey the repository and measure various dimensions of scale. + +As repositories grow to "monorepo" size, certain data shapes can cause +performance problems. `git-survey` attempts to measure and report on +known problem areas. + +Ref Selection and Reachable Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this first analysis phase, `git survey` will iterate over the set of +requested branches, tags, and other refs and treewalk over all of the +reachable commits, trees, and blobs and generate various statistics. + +OPTIONS +------- + +--progress:: + Show progress. This is automatically enabled when interactive. + +Ref Selection +~~~~~~~~~~~~~ + +The following options control the set of refs that `git survey` will examine. +By default, `git survey` will look at tags, local branches, and remote refs. +If any of the following options are given, the default set is cleared and +only refs for the given options are added. + +--all-refs:: + Use all refs. This includes local branches, tags, remote refs, + notes, and stashes. This option overrides all of the following. + +--branches:: + Add local branches (`refs/heads/`) to the set. + +--tags:: + Add tags (`refs/tags/`) to the set. + +--remotes:: + Add remote branches (`refs/remote/`) to the set. + +--detached:: + Add HEAD to the set. + +--other:: + Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set. + +OUTPUT +------ + +By default, `git survey` will print information about the repository in a +human-readable format that includes overviews and tables. + +References Summary +~~~~~~~~~~~~~~~~~~ + +The references summary includes a count of each kind of reference, +including branches, remote refs, and tags (split by "all" and +"annotated"). + +Reachable Object Summary +~~~~~~~~~~~~~~~~~~~~~~~~ + +The reachable object summary shows the total number of each kind of Git +object, including tags, commits, trees, and blobs. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index bd76167a45eb71..579d9940f62952 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -403,6 +403,36 @@ sign `$` upon checkout. Any byte sequence that begins with with `$Id$` upon check-in. +`symlink` +^^^^^^^^^ + +On Windows, symbolic links have a type: a "file symlink" must point at +a file, and a "directory symlink" must point at a directory. If the +type of symlink does not match its target, it doesn't work. + +Git does not record the type of symlink in the index or in a tree. On +checkout it'll guess the type, which only works if the target exists +at the time the symlink is created. This may often not be the case, +for example when the link points at a directory inside a submodule. + +The `symlink` attribute allows you to explicitly set the type of symlink +to `file` or `dir`, so Git doesn't have to guess. If you have a set of +symlinks that point at other files, you can do: + +------------------------ +*.gif symlink=file +------------------------ + +To tell Git that a symlink points at a directory, use: + +------------------------ +tools_folder symlink=dir +------------------------ + +The `symlink` attribute is ignored on platforms other than Windows, +since they don't distinguish between different types of symlinks. + + `filter` ^^^^^^^^ diff --git a/Documentation/meson.build b/Documentation/meson.build index f4854f802d455f..605c0b5673205a 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -145,6 +145,7 @@ manpages = { 'git-status.adoc' : 1, 'git-stripspace.adoc' : 1, 'git-submodule.adoc' : 1, + 'git-survey.adoc' : 1, 'git-svn.adoc' : 1, 'git-switch.adoc' : 1, 'git-symbolic-ref.adoc' : 1, diff --git a/Makefile b/Makefile index b31ecb07564a73..a9c0f262a3b64f 100644 --- a/Makefile +++ b/Makefile @@ -479,6 +479,11 @@ include shared.mak # # CURL_LDFLAGS=-lcurl # +# Define LAZYLOAD_LIBCURL to dynamically load the libcurl; This can be useful +# if Multiple libcurl versions exist (with different file names) that link to +# various SSL/TLS backends, to support the `http.sslBackend` runtime switch in +# such a scenario. +# # === Optional library: libpcre2 === # # Define USE_LIBPCRE if you have and want to use libpcre. Various @@ -828,6 +833,7 @@ TEST_BUILTINS_OBJS += test-hash-speed.o TEST_BUILTINS_OBJS += test-hash.o TEST_BUILTINS_OBJS += test-hashmap.o TEST_BUILTINS_OBJS += test-hexdump.o +TEST_BUILTINS_OBJS += test-iconv.o TEST_BUILTINS_OBJS += test-json-writer.o TEST_BUILTINS_OBJS += test-lazy-init-name-hash.o TEST_BUILTINS_OBJS += test-match-trees.o @@ -1488,6 +1494,7 @@ BUILTIN_OBJS += builtin/sparse-checkout.o BUILTIN_OBJS += builtin/stash.o BUILTIN_OBJS += builtin/stripspace.o BUILTIN_OBJS += builtin/submodule--helper.o +BUILTIN_OBJS += builtin/survey.o BUILTIN_OBJS += builtin/symbolic-ref.o BUILTIN_OBJS += builtin/tag.o BUILTIN_OBJS += builtin/unpack-file.o @@ -1527,6 +1534,7 @@ CLAR_TEST_SUITES += u-hash CLAR_TEST_SUITES += u-hashmap CLAR_TEST_SUITES += u-list-objects-filter-options CLAR_TEST_SUITES += u-mem-pool +CLAR_TEST_SUITES += u-mingw CLAR_TEST_SUITES += u-odb-inmemory CLAR_TEST_SUITES += u-oid-array CLAR_TEST_SUITES += u-oidmap @@ -1789,10 +1797,23 @@ else CURL_LIBCURL = endif - ifndef CURL_LDFLAGS - CURL_LDFLAGS = $(eval CURL_LDFLAGS := $$(shell $$(CURL_CONFIG) --libs))$(CURL_LDFLAGS) + ifdef LAZYLOAD_LIBCURL + LAZYLOAD_LIBCURL_OBJ = compat/lazyload-curl.o + OBJECTS += $(LAZYLOAD_LIBCURL_OBJ) + # The `CURL_STATICLIB` constant must be defined to avoid seeing the functions + # declared as DLL imports + CURL_CFLAGS = -DCURL_STATICLIB +ifneq ($(uname_S),MINGW) +ifneq ($(uname_S),Windows) + CURL_LIBCURL = -ldl +endif +endif + else + ifndef CURL_LDFLAGS + CURL_LDFLAGS = $(eval CURL_LDFLAGS := $$(shell $$(CURL_CONFIG) --libs))$(CURL_LDFLAGS) + endif + CURL_LIBCURL += $(CURL_LDFLAGS) endif - CURL_LIBCURL += $(CURL_LDFLAGS) ifndef CURL_CFLAGS CURL_CFLAGS = $(eval CURL_CFLAGS := $$(shell $$(CURL_CONFIG) --cflags))$(CURL_CFLAGS) @@ -1813,7 +1834,7 @@ else endif ifdef USE_CURL_FOR_IMAP_SEND BASIC_CFLAGS += -DUSE_CURL_FOR_IMAP_SEND - IMAP_SEND_BUILDDEPS = http.o + IMAP_SEND_BUILDDEPS = http.o $(LAZYLOAD_LIBCURL_OBJ) IMAP_SEND_LDFLAGS += $(CURL_LIBCURL) endif ifndef NO_EXPAT @@ -2992,10 +3013,10 @@ git-imap-send$X: imap-send.o $(IMAP_SEND_BUILDDEPS) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(IMAP_SEND_LDFLAGS) $(LIBS) -git-http-fetch$X: http.o http-walker.o http-fetch.o GIT-LDFLAGS $(GITLIBS) +git-http-fetch$X: http.o http-walker.o http-fetch.o $(LAZYLOAD_LIBCURL_OBJ) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(LIBS) -git-http-push$X: http.o http-push.o GIT-LDFLAGS $(GITLIBS) +git-http-push$X: http.o http-push.o $(LAZYLOAD_LIBCURL_OBJ) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) @@ -3005,7 +3026,7 @@ $(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY) ln -s $< $@ 2>/dev/null || \ cp $< $@ -$(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS) +$(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OBJ) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) @@ -3900,12 +3921,15 @@ ifdef MSVC $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) $(RM) headless-git.o.pdb $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ilk,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.ilk,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ilk,$(TEST_PROGRAMS)) $(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS)) $(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS)) $(RM) compat/vcbuild/MSVC-DEFS-GEN diff --git a/README.md b/README.md index d87bca1b8c3ebf..026d5d85caef09 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,77 @@ -[![Build status](https://github.com/git/git/workflows/CI/badge.svg)](https://github.com/git/git/actions?query=branch%3Amaster+event%3Apush) +Git for Windows +=============== + +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) +[![Open in Visual Studio Code](https://img.shields.io/static/v1?logo=visualstudiocode&label=&message=Open%20in%20Visual%20Studio%20Code&labelColor=2c2c32&color=007acc&logoColor=007acc)](https://open.vscode.dev/git-for-windows/git) +[![Build status](https://github.com/git-for-windows/git/workflows/CI/badge.svg)](https://github.com/git-for-windows/git/actions?query=branch%3Amain+event%3Apush) +[![Join the chat at https://gitter.im/git-for-windows/git](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +This is [Git for Windows](http://git-for-windows.github.io/), the Windows port +of [Git](http://git-scm.com/). + +The Git for Windows project is run using a [governance +model](http://git-for-windows.github.io/governance-model.html). If you +encounter problems, you can report them as [GitHub +issues](https://github.com/git-for-windows/git/issues), discuss them in Git +for Windows' [Discussions](https://github.com/git-for-windows/git/discussions) +or on the [Git mailing list](mailto:git@vger.kernel.org), and [contribute bug +fixes](https://gitforwindows.org/how-to-participate). + +To build Git for Windows, please either install [Git for Windows' +SDK](https://gitforwindows.org/#download-sdk), start its `git-bash.exe`, `cd` +to your Git worktree and run `make`, or open the Git worktree as a folder in +Visual Studio. + +To verify that your build works, use one of the following methods: + +- If you want to test the built executables within Git for Windows' SDK, + prepend `/bin-wrappers` to the `PATH`. +- Alternatively, run `make install` in the Git worktree. +- If you need to test this in a full installer, run `sdk build + git-and-installer`. +- You can also "install" Git into an existing portable Git via `make install + DESTDIR=` where `` refers to the top-level directory of the + portable Git. In this instance, you will want to prepend that portable Git's + `/cmd` directory to the `PATH`, or test by running that portable Git's + `git-bash.exe` or `git-cmd.exe`. +- If you built using a recent Visual Studio, you can use the menu item + `Build>Install git` (you will want to click on `Project>CMake Settings for + Git` first, then click on `Edit JSON` and then point `installRoot` to the + `mingw64` directory of an already-unpacked portable Git). + + As in the previous bullet point, you will then prepend `/cmd` to the `PATH` + or run using the portable Git's `git-bash.exe` or `git-cmd.exe`. +- If you want to run the built executables in-place, but in a CMD instead of + inside a Bash, you can run a snippet like this in the `git-bash.exe` window + where Git was built (ensure that the `EOF` line has no leading spaces), and + then paste into the CMD window what was put in the clipboard: + + ```sh + clip.exe < (see https://subspace.kernel.org/subscribing.html for details). The mailing list archives are available at , and other archival sites. +The core git mailing list is plain text (no HTML!). Issues which are security relevant should be disclosed privately to the Git Security mailing list . diff --git a/SECURITY.md b/SECURITY.md index c720c2ae7f9580..42b6d458bfd557 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,24 +28,38 @@ Examples for details to include: ## Supported Versions -There are no official "Long Term Support" versions in Git. -Instead, the maintenance track (i.e. the versions based on the -most recently published feature release, also known as ".0" -version) sees occasional updates with bug fixes. - -Fixes to vulnerabilities are made for the maintenance track for -the latest feature release and merged up to the in-development -branches. The Git project makes no formal guarantee for any -older maintenance tracks to receive updates. In practice, -though, critical vulnerability fixes are applied not only to the -most recent track, but to at least a couple more maintenance -tracks. - -This is typically done by making the fix on the oldest and still -relevant maintenance track, and merging it upwards to newer and -newer maintenance tracks. - -For example, v2.24.1 was released to address a couple of -[CVEs](https://cve.mitre.org/), and at the same time v2.14.6, -v2.15.4, v2.16.6, v2.17.3, v2.18.2, v2.19.3, v2.20.2, v2.21.1, -v2.22.2 and v2.23.1 were released. +Git for Windows is a "friendly fork" of [Git](https://git-scm.com/), i.e. changes in Git for Windows are frequently contributed back, and Git for Windows' release cycle closely following Git's. + +While Git maintains several release trains (when v2.19.1 was released, there were updates to v2.14.x-v2.18.x, too, for example), Git for Windows follows only the latest Git release. For example, there is no Git for Windows release corresponding to Git v2.16.5 (which was released after v2.19.0). + +One exception is [MinGit for Windows](https://gitforwindows.org/mingit) (a minimal subset of Git for Windows, intended for bundling with third-party applications that do not need any interactive commands nor support for `git svn`): critical security fixes are backported to the v2.11.x, v2.14.x, v2.19.x, v2.21.x and v2.23.x release trains. + +## Version number scheme + +The Git for Windows versions reflect the Git version on which they are based. For example, Git for Windows v2.21.0 is based on Git v2.21.0. + +As Git for Windows bundles more than just Git (such as Bash, OpenSSL, OpenSSH, GNU Privacy Guard), sometimes there are interim releases without corresponding Git releases. In these cases, Git for Windows appends a number in parentheses, starting with the number 2, then 3, etc. For example, both Git for Windows v2.17.1 and v2.17.1(2) were based on Git v2.17.1, but the latter included updates for Git Credential Manager and Git LFS, fixing critical regressions. + +## Tag naming scheme + +Every Git for Windows version is tagged using a name that starts with the Git version on which it is based, with the suffix `.windows.` appended. For example, Git for Windows v2.17.1' source code is tagged as [`v2.17.1.windows.1`](https://github.com/git-for-windows/git/releases/tag/v2.17.1.windows.1) (the patch level is always at least 1, given that Git for Windows always has patches on top of Git). Likewise, Git for Windows v2.17.1(2)' source code is tagged as [`v2.17.1.windows.2`](https://github.com/git-for-windows/git/releases/tag/v2.17.1.windows.2). + +## Release Candidate (rc) versions + +As a friendly fork of Git (the "upstream" project), Git for Windows is closely corelated to that project. + +Consequently, Git for Windows publishes versions based on Git's release candidates (for upcoming "`.0`" versions, see [Git's release schedule](https://tinyurl.com/gitCal)). These versions end in `-rc`, starting with `-rc0` for a very early preview of what is to come, and as with regular versions, Git for Windows tries to follow Git's releases as quickly as possible. + +Note: there is currently a bug in the "Check daily for updates" code, where it mistakes the final version as a downgrade from release candidates. Example: if you installed Git for Windows v2.23.0-rc3 and enabled the auto-updater, it would ask you whether you want to "downgrade" to v2.23.0 when that version was available. + +[All releases](https://github.com/git-for-windows/git/releases/), including release candidates, are listed via a link at the footer of the [Git for Windows](https://gitforwindows.org/) home page. + +## Snapshot versions ('nightly builds') + +Git for Windows also provides snapshots (these are not releases) of the current development as per git-for-Windows/git's `master` branch at the [Snapshots](https://gitforwindows.org/git-snapshots/) page. This link is also listed in the footer of the [Git for Windows](https://gitforwindows.org/) home page. + +Note: even if those builds are not exactly "nightly", they are sometimes referred to as "nightly builds" to keep with other projects' nomenclature. + +## Following upstream's developments + +The [gitforwindows/git repository](https://github.com/git-for-windows/git) also provides the `shears/*` branches. The `shears/*` branches reflect Git for Windows' patches, rebased onto the upstream integration branches, [updated (mostly) via automated CI builds](https://dev.azure.com/git-for-windows/git/_build?definitionId=25). diff --git a/abspath.c b/abspath.c index 1202cde23dbc9b..0c17e98654e4b0 100644 --- a/abspath.c +++ b/abspath.c @@ -93,6 +93,9 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path, goto error_out; } + if (platform_strbuf_realpath(resolved, path)) + return resolved->buf; + strbuf_addstr(&remaining, path); get_root_part(resolved, &remaining); diff --git a/advice.c b/advice.c index 0018501b7bc103..71ddedd4ad46bb 100644 --- a/advice.c +++ b/advice.c @@ -61,6 +61,7 @@ static struct { [ADVICE_IGNORED_HOOK] = { "ignoredHook" }, [ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity" }, [ADVICE_MERGE_CONFLICT] = { "mergeConflict" }, + [ADVICE_NAME_TOO_LONG] = { "nameTooLong" }, [ADVICE_NESTED_TAG] = { "nestedTag" }, [ADVICE_OBJECT_NAME_WARNING] = { "objectNameWarning" }, [ADVICE_PUSH_ALREADY_EXISTS] = { "pushAlreadyExists" }, @@ -89,6 +90,7 @@ static struct { [ADVICE_SUBMODULE_MERGE_CONFLICT] = { "submoduleMergeConflict" }, [ADVICE_SUGGEST_DETACHING_HEAD] = { "suggestDetachingHead" }, [ADVICE_UPDATE_SPARSE_PATH] = { "updateSparsePath" }, + [ADVICE_USE_CORE_FSMONITOR_CONFIG] = { "useCoreFSMonitorConfig" }, [ADVICE_WAITING_FOR_EDITOR] = { "waitingForEditor" }, [ADVICE_WORKTREE_ADD_ORPHAN] = { "worktreeAddOrphan" }, }; diff --git a/advice.h b/advice.h index 8def28068861df..849a5991379c11 100644 --- a/advice.h +++ b/advice.h @@ -28,6 +28,7 @@ enum advice_type { ADVICE_IGNORED_HOOK, ADVICE_IMPLICIT_IDENTITY, ADVICE_MERGE_CONFLICT, + ADVICE_NAME_TOO_LONG, ADVICE_NESTED_TAG, ADVICE_OBJECT_NAME_WARNING, ADVICE_PUSH_ALREADY_EXISTS, @@ -56,6 +57,7 @@ enum advice_type { ADVICE_SUBMODULE_MERGE_CONFLICT, ADVICE_SUGGEST_DETACHING_HEAD, ADVICE_UPDATE_SPARSE_PATH, + ADVICE_USE_CORE_FSMONITOR_CONFIG, ADVICE_WAITING_FOR_EDITOR, ADVICE_WORKTREE_ADD_ORPHAN, }; diff --git a/apply.c b/apply.c index 249248d4f205ca..749b515ffcc66e 100644 --- a/apply.c +++ b/apply.c @@ -4515,7 +4515,7 @@ static int try_create_file(struct apply_state *state, const char *path, /* Although buf:size is counted string, it also is NUL * terminated. */ - return !!symlink(buf, path); + return !!create_symlink(state && state->repo ? state->repo->index : NULL, buf, path); fd = open(path, O_CREAT | O_EXCL | O_WRONLY, (mode & 0100) ? 0777 : 0666); if (fd < 0) diff --git a/builtin.h b/builtin.h index 4e47a4ebd30ba3..d3caec75424f71 100644 --- a/builtin.h +++ b/builtin.h @@ -260,6 +260,7 @@ int cmd_sparse_checkout(int argc, const char **argv, const char *prefix, struct int cmd_status(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_stash(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_stripspace(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_submodule__helper(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_switch(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_symbolic_ref(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/add.c b/builtin/add.c index c859f665199efa..46e0bfe4209566 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -498,6 +498,10 @@ int cmd_add(int argc, die_in_unpopulated_submodule(repo->index, prefix); die_path_inside_submodule(repo->index, &pathspec); + enable_fscache(0); + /* We do not really re-read the index but update the up-to-date flags */ + preload_index(repo->index, &pathspec, 0); + if (add_new_files) { int baselen; @@ -610,5 +614,6 @@ int cmd_add(int argc, free(ps_matched); dir_clear(&dir); clear_pathspec(&pathspec); + enable_fscache(0); return exit_status; } diff --git a/builtin/checkout.c b/builtin/checkout.c index f82adcb740287e..632a2f64582f87 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -410,6 +410,7 @@ static int checkout_worktree(const struct checkout_opts *opts, if (pc_workers > 1) init_parallel_checkout(); + enable_fscache(the_repository->index->cache_nr); for (pos = 0; pos < the_repository->index->cache_nr; pos++) { struct cache_entry *ce = the_repository->index->cache[pos]; if (ce->ce_flags & CE_MATCHED) { @@ -435,6 +436,7 @@ static int checkout_worktree(const struct checkout_opts *opts, errs |= run_parallel_checkout(&state, pc_workers, pc_threshold, NULL, NULL); mem_pool_discard(&ce_mem_pool, should_validate_cache_entries()); + disable_fscache(); remove_marked_cache_entries(the_repository->index, 1); remove_scheduled_dirs(); errs |= finish_delayed_checkout(&state, opts->show_progress); diff --git a/builtin/clean.c b/builtin/clean.c index 1d5e7e5366bf09..f8a54a4a47bc7b 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -26,6 +26,7 @@ #include "pathspec.h" #include "help.h" #include "prompt.h" +#include "advice.h" static int require_force = -1; /* unset */ static int interactive; @@ -41,6 +42,10 @@ static const char *msg_remove = N_("Removing %s\n"); static const char *msg_would_remove = N_("Would remove %s\n"); static const char *msg_skip_git_dir = N_("Skipping repository %s\n"); static const char *msg_would_skip_git_dir = N_("Would skip repository %s\n"); +#ifndef CAN_UNLINK_MOUNT_POINTS +static const char *msg_skip_mount_point = N_("Skipping mount point %s\n"); +static const char *msg_would_skip_mount_point = N_("Would skip mount point %s\n"); +#endif static const char *msg_warn_remove_failed = N_("failed to remove %s"); static const char *msg_warn_lstat_failed = N_("could not lstat %s\n"); static const char *msg_skip_cwd = N_("Refusing to remove current working directory\n"); @@ -185,6 +190,29 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, goto out; } + if (is_mount_point(path)) { +#ifndef CAN_UNLINK_MOUNT_POINTS + if (!quiet) { + quote_path(path->buf, prefix, "ed, 0); + printf(dry_run ? + _(msg_would_skip_mount_point) : + _(msg_skip_mount_point), quoted.buf); + } + *dir_gone = 0; +#else + if (!dry_run && unlink(path->buf)) { + int saved_errno = errno; + quote_path(path->buf, prefix, "ed, 0); + errno = saved_errno; + warning_errno(_(msg_warn_remove_failed), quoted.buf); + *dir_gone = 0; + ret = -1; + } +#endif + + goto out; + } + dir = opendir(path->buf); if (!dir) { /* an empty dir could be removed even if it is unreadble */ @@ -194,6 +222,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, quote_path(path->buf, prefix, "ed, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), quoted.buf); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } *dir_gone = 0; } ret = res; @@ -229,6 +260,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, quote_path(path->buf, prefix, "ed, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), quoted.buf); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } *dir_gone = 0; ret = 1; } @@ -272,6 +306,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, quote_path(path->buf, prefix, "ed, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), quoted.buf); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } *dir_gone = 0; ret = 1; } @@ -1015,6 +1052,7 @@ int cmd_clean(int argc, if (repo_read_index(the_repository) < 0) die(_("index file corrupt")); + enable_fscache(the_repository->index->cache_nr); pl = add_pattern_list(&dir, EXC_CMDL, "--exclude option"); for (i = 0; i < exclude_list.nr; i++) @@ -1081,6 +1119,9 @@ int cmd_clean(int argc, qname = quote_path(item->string, NULL, &buf, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), qname); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } errors++; } else if (!quiet) { qname = quote_path(item->string, NULL, &buf, 0); @@ -1089,6 +1130,7 @@ int cmd_clean(int argc, } } + disable_fscache(); strbuf_release(&abs_path); strbuf_release(&buf); string_list_clear(&del_list, 0); diff --git a/builtin/commit.c b/builtin/commit.c index 28f61745034506..3894b05e7cdf8b 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1623,6 +1623,7 @@ struct repository *repo UNUSED) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; @@ -1663,6 +1664,7 @@ struct repository *repo UNUSED) wt_status_print(&s); wt_status_collect_free_buffers(&s); + disable_fscache(); return 0; } diff --git a/builtin/credential-cache.c b/builtin/credential-cache.c index 7f733cb756e03c..3b8130d3d64f9c 100644 --- a/builtin/credential-cache.c +++ b/builtin/credential-cache.c @@ -23,7 +23,7 @@ static int connection_closed(int error) static int connection_fatally_broken(int error) { - return (error != ENOENT) && (error != ENETDOWN); + return (error != ENOENT) && (error != ENETDOWN) && (error != ECONNREFUSED); } #else diff --git a/builtin/difftool.c b/builtin/difftool.c index 2a21005f2ee264..35e363c05f4832 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -544,7 +544,7 @@ static int run_dir_diff(struct repository *repo, } add_path(&wtdir, wtdir_len, dst_path); if (dt_options->symlinks) { - if (symlink(wtdir.buf, rdir.buf)) { + if (create_symlink(lstate.istate, wtdir.buf, rdir.buf)) { ret = error_errno("could not symlink '%s' to '%s'", wtdir.buf, rdir.buf); goto finish; } diff --git a/builtin/reset.c b/builtin/reset.c index 3be6bd0121afe5..8e3c8509d73b77 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -38,6 +38,8 @@ #include "trace2.h" #include "dir.h" #include "add-interactive.h" +#include "strbuf.h" +#include "quote.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -46,6 +48,7 @@ static const char * const git_reset_usage[] = { N_("git reset [-q] [] [--] ..."), N_("git reset [-q] [--pathspec-from-file [--pathspec-file-nul]] []"), N_("git reset --patch [] [--] [...]"), + N_("DEPRECATED: git reset [-q] [--stdin [-z]] []"), NULL }; @@ -347,6 +350,7 @@ int cmd_reset(int argc, struct pathspec pathspec; int intent_to_add = 0; struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; + int nul_term_line = 0, read_from_stdin = 0; const struct option options[] = { OPT__QUIET(&quiet, N_("be quiet, only report errors")), OPT_BOOL(0, "no-refresh", &no_refresh, @@ -379,6 +383,10 @@ int cmd_reset(int argc, N_("record only the fact that removed paths will be added later")), OPT_PATHSPEC_FROM_FILE(&pathspec_from_file), OPT_PATHSPEC_FILE_NUL(&pathspec_file_nul), + OPT_BOOL('z', NULL, &nul_term_line, + N_("DEPRECATED (use --pathspec-file-nul instead): paths are separated with NUL character")), + OPT_BOOL(0, "stdin", &read_from_stdin, + N_("DEPRECATED (use --pathspec-from-file=- instead): read paths from ")), OPT_END() }; @@ -388,6 +396,14 @@ int cmd_reset(int argc, PARSE_OPT_KEEP_DASHDASH); parse_args(&pathspec, argv, prefix, patch_mode, &rev); + if (read_from_stdin) { + warning(_("--stdin is deprecated, please use --pathspec-from-file=- instead")); + free(pathspec_from_file); + pathspec_from_file = xstrdup("-"); + if (nul_term_line) + pathspec_file_nul = 1; + } + if (pathspec_from_file) { if (patch_mode) die(_("options '%s' and '%s' cannot be used together"), "--pathspec-from-file", "--patch"); diff --git a/builtin/survey.c b/builtin/survey.c new file mode 100644 index 00000000000000..f40905fb2fd57a --- /dev/null +++ b/builtin/survey.c @@ -0,0 +1,934 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "builtin.h" +#include "config.h" +#include "environment.h" +#include "hex.h" +#include "object.h" +#include "odb.h" +#include "object-name.h" +#include "parse-options.h" +#include "path-walk.h" +#include "progress.h" +#include "ref-filter.h" +#include "refs.h" +#include "revision.h" +#include "strbuf.h" +#include "strvec.h" +#include "tag.h" +#include "trace2.h" +#include "color.h" + +static const char * const survey_usage[] = { + N_("(EXPERIMENTAL!) git survey "), + NULL, +}; + +struct survey_refs_wanted { + int want_all_refs; /* special override */ + + int want_branches; + int want_tags; + int want_remotes; + int want_detached; + int want_other; /* see FILTER_REFS_OTHERS -- refs/notes/, refs/stash/ */ +}; + +static struct survey_refs_wanted default_ref_options = { + .want_all_refs = 1, +}; + +struct survey_opts { + int verbose; + int show_progress; + int top_nr; + struct survey_refs_wanted refs; +}; + +struct survey_report_ref_summary { + size_t refs_nr; + size_t branches_nr; + size_t remote_refs_nr; + size_t tags_nr; + size_t tags_annotated_nr; + size_t others_nr; + size_t unknown_nr; +}; + +struct survey_report_object_summary { + size_t commits_nr; + size_t tags_nr; + size_t trees_nr; + size_t blobs_nr; +}; + +/** + * For some category given by 'label', count the number of objects + * that match that label along with the on-disk size and the size + * after decompressing (both with delta bases and zlib). + */ +struct survey_report_object_size_summary { + char *label; + size_t nr; + size_t disk_size; + size_t inflated_size; + size_t num_missing; +}; + +typedef int (*survey_top_cmp)(void *v1, void *v2); + +static int cmp_by_nr(void *v1, void *v2) +{ + struct survey_report_object_size_summary *s1 = v1; + struct survey_report_object_size_summary *s2 = v2; + + if (s1->nr < s2->nr) + return -1; + if (s1->nr > s2->nr) + return 1; + return 0; +} + +static int cmp_by_disk_size(void *v1, void *v2) +{ + struct survey_report_object_size_summary *s1 = v1; + struct survey_report_object_size_summary *s2 = v2; + + if (s1->disk_size < s2->disk_size) + return -1; + if (s1->disk_size > s2->disk_size) + return 1; + return 0; +} + +static int cmp_by_inflated_size(void *v1, void *v2) +{ + struct survey_report_object_size_summary *s1 = v1; + struct survey_report_object_size_summary *s2 = v2; + + if (s1->inflated_size < s2->inflated_size) + return -1; + if (s1->inflated_size > s2->inflated_size) + return 1; + return 0; +} + +/** + * Store a list of "top" categories by some sorting function. When + * inserting a new category, reorder the list and free the one that + * got ejected (if any). + */ +struct survey_report_top_table { + const char *name; + survey_top_cmp cmp_fn; + size_t nr; + size_t alloc; + + /** + * 'data' stores an array of structs and must be cast into + * the proper array type before evaluating an index. + */ + void *data; +}; + +static void init_top_sizes(struct survey_report_top_table *top, + size_t limit, const char *name, + survey_top_cmp cmp) +{ + struct survey_report_object_size_summary *sz_array; + + top->name = name; + top->cmp_fn = cmp; + top->alloc = limit; + top->nr = 0; + + CALLOC_ARRAY(sz_array, limit); + top->data = sz_array; +} + +MAYBE_UNUSED +static void clear_top_sizes(struct survey_report_top_table *top) +{ + struct survey_report_object_size_summary *sz_array = top->data; + + for (size_t i = 0; i < top->nr; i++) + free(sz_array[i].label); + free(sz_array); +} + +static void maybe_insert_into_top_size(struct survey_report_top_table *top, + struct survey_report_object_size_summary *summary) +{ + struct survey_report_object_size_summary *sz_array = top->data; + size_t pos = top->nr; + + /* Compare against list from the bottom. */ + while (pos > 0 && top->cmp_fn(&sz_array[pos - 1], summary) < 0) + pos--; + + /* Not big enough! */ + if (pos >= top->alloc) + return; + + /* We need to shift the data. */ + if (top->nr == top->alloc) + free(sz_array[top->nr - 1].label); + else + top->nr++; + + for (size_t i = top->nr - 1; i > pos; i--) + memcpy(&sz_array[i], &sz_array[i - 1], sizeof(*sz_array)); + + memcpy(&sz_array[pos], summary, sizeof(*summary)); + sz_array[pos].label = xstrdup(summary->label); +} + +/** + * This struct contains all of the information that needs to be printed + * at the end of the exploration of the repository and its references. + */ +struct survey_report { + struct survey_report_ref_summary refs; + struct survey_report_object_summary reachable_objects; + + struct survey_report_object_size_summary *by_type; + + struct survey_report_top_table *top_paths_by_count; + struct survey_report_top_table *top_paths_by_disk; + struct survey_report_top_table *top_paths_by_inflate; +}; + +#define REPORT_TYPE_COMMIT 0 +#define REPORT_TYPE_TREE 1 +#define REPORT_TYPE_BLOB 2 +#define REPORT_TYPE_TAG 3 +#define REPORT_TYPE_COUNT 4 + +struct survey_context { + struct repository *repo; + + /* Options that control what is done. */ + struct survey_opts opts; + + /* Info for output only. */ + struct survey_report report; + + /* + * The rest of the members are about enabling the activity + * of the 'git survey' command, including ref listings, object + * pointers, and progress. + */ + + struct progress *progress; + size_t progress_nr; + size_t progress_total; + + struct strvec refs; + struct ref_array ref_array; +}; + +static void clear_survey_context(struct survey_context *ctx) +{ + ref_array_clear(&ctx->ref_array); + strvec_clear(&ctx->refs); +} + +struct survey_table { + const char *table_name; + struct strvec header; + struct strvec *rows; + size_t rows_nr; + size_t rows_alloc; +}; + +#define SURVEY_TABLE_INIT { \ + .header = STRVEC_INIT, \ +} + +static void clear_table(struct survey_table *table) +{ + strvec_clear(&table->header); + for (size_t i = 0; i < table->rows_nr; i++) + strvec_clear(&table->rows[i]); + free(table->rows); +} + +static void insert_table_rowv(struct survey_table *table, ...) +{ + va_list ap; + char *arg; + ALLOC_GROW(table->rows, table->rows_nr + 1, table->rows_alloc); + + memset(&table->rows[table->rows_nr], 0, sizeof(struct strvec)); + + va_start(ap, table); + while ((arg = va_arg(ap, char *))) + strvec_push(&table->rows[table->rows_nr], arg); + va_end(ap); + + table->rows_nr++; +} + +#define SECTION_SEGMENT "========================================" +#define SECTION_SEGMENT_LEN 40 +static const char *section_line = SECTION_SEGMENT + SECTION_SEGMENT + SECTION_SEGMENT + SECTION_SEGMENT; +static const size_t section_len = 4 * SECTION_SEGMENT_LEN; + +static void print_table_title(const char *name, size_t *widths, size_t nr) +{ + size_t width = 3 * (nr - 1); + size_t min_width = strlen(name); + + for (size_t i = 0; i < nr; i++) + width += widths[i]; + + if (width < min_width) + width = min_width; + + if (width > section_len) + width = section_len; + + printf("\n%s\n%.*s\n", name, (int)width, section_line); +} + +static void print_row_plaintext(struct strvec *row, size_t *widths) +{ + static struct strbuf line = STRBUF_INIT; + strbuf_setlen(&line, 0); + + for (size_t i = 0; i < row->nr; i++) { + const char *str = row->v[i]; + size_t len = strlen(str); + if (i) + strbuf_add(&line, " | ", 3); + strbuf_addchars(&line, ' ', widths[i] - len); + strbuf_add(&line, str, len); + } + printf("%s\n", line.buf); +} + +static void print_divider_plaintext(size_t *widths, size_t nr) +{ + static struct strbuf line = STRBUF_INIT; + strbuf_setlen(&line, 0); + + for (size_t i = 0; i < nr; i++) { + if (i) + strbuf_add(&line, "-+-", 3); + strbuf_addchars(&line, '-', widths[i]); + } + printf("%s\n", line.buf); +} + +static void print_table_plaintext(struct survey_table *table) +{ + size_t *column_widths; + size_t columns_nr = table->header.nr; + CALLOC_ARRAY(column_widths, columns_nr); + + for (size_t i = 0; i < columns_nr; i++) { + column_widths[i] = strlen(table->header.v[i]); + + for (size_t j = 0; j < table->rows_nr; j++) { + size_t rowlen = strlen(table->rows[j].v[i]); + if (column_widths[i] < rowlen) + column_widths[i] = rowlen; + } + } + + print_table_title(table->table_name, column_widths, columns_nr); + print_row_plaintext(&table->header, column_widths); + print_divider_plaintext(column_widths, columns_nr); + + for (size_t j = 0; j < table->rows_nr; j++) + print_row_plaintext(&table->rows[j], column_widths); + + free(column_widths); +} + +static void survey_report_plaintext_refs(struct survey_context *ctx) +{ + struct survey_report_ref_summary *refs = &ctx->report.refs; + struct survey_table table = SURVEY_TABLE_INIT; + + table.table_name = _("REFERENCES SUMMARY"); + + strvec_push(&table.header, _("Ref Type")); + strvec_push(&table.header, _("Count")); + + if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_branches) { + char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->branches_nr); + insert_table_rowv(&table, _("Branches"), fmt, NULL); + free(fmt); + } + + if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_remotes) { + char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->remote_refs_nr); + insert_table_rowv(&table, _("Remote refs"), fmt, NULL); + free(fmt); + } + + if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_tags) { + char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->tags_nr); + insert_table_rowv(&table, _("Tags (all)"), fmt, NULL); + free(fmt); + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->tags_annotated_nr); + insert_table_rowv(&table, _("Tags (annotated)"), fmt, NULL); + free(fmt); + } + + print_table_plaintext(&table); + clear_table(&table); +} + +static void survey_report_plaintext_reachable_object_summary(struct survey_context *ctx) +{ + struct survey_report_object_summary *objs = &ctx->report.reachable_objects; + struct survey_table table = SURVEY_TABLE_INIT; + char *fmt; + + table.table_name = _("REACHABLE OBJECT SUMMARY"); + + strvec_push(&table.header, _("Object Type")); + strvec_push(&table.header, _("Count")); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->tags_nr); + insert_table_rowv(&table, _("Tags"), fmt, NULL); + free(fmt); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->commits_nr); + insert_table_rowv(&table, _("Commits"), fmt, NULL); + free(fmt); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->trees_nr); + insert_table_rowv(&table, _("Trees"), fmt, NULL); + free(fmt); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->blobs_nr); + insert_table_rowv(&table, _("Blobs"), fmt, NULL); + free(fmt); + + print_table_plaintext(&table); + clear_table(&table); +} + +static void survey_report_object_sizes(const char *title, + const char *categories, + struct survey_report_object_size_summary *summary, + size_t summary_nr) +{ + struct survey_table table = SURVEY_TABLE_INIT; + table.table_name = title; + + strvec_push(&table.header, categories); + strvec_push(&table.header, _("Count")); + strvec_push(&table.header, _("Disk Size")); + strvec_push(&table.header, _("Inflated Size")); + + for (size_t i = 0; i < summary_nr; i++) { + char *label_str = xstrdup(summary[i].label); + char *nr_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].nr); + char *disk_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].disk_size); + char *inflate_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].inflated_size); + + insert_table_rowv(&table, label_str, nr_str, + disk_str, inflate_str, NULL); + + free(label_str); + free(nr_str); + free(disk_str); + free(inflate_str); + } + + print_table_plaintext(&table); + clear_table(&table); +} + +static void survey_report_plaintext_sorted_size( + struct survey_report_top_table *top) +{ + survey_report_object_sizes(top->name, _("Path"), + top->data, top->nr); +} + +static void survey_report_plaintext(struct survey_context *ctx) +{ + printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree); + printf("-----------------------------------------------------\n"); + survey_report_plaintext_refs(ctx); + survey_report_plaintext_reachable_object_summary(ctx); + survey_report_object_sizes(_("TOTAL OBJECT SIZES BY TYPE"), + _("Object Type"), + ctx->report.by_type, + REPORT_TYPE_COUNT); + + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_count[REPORT_TYPE_TREE]); + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_count[REPORT_TYPE_BLOB]); + + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_disk[REPORT_TYPE_TREE]); + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB]); + + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE]); + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB]); +} + +/* + * After parsing the command line arguments, figure out which refs we + * should scan. + * + * If ANY were given in positive sense, then we ONLY include them and + * do not use the builtin values. + */ +static void fixup_refs_wanted(struct survey_context *ctx) +{ + struct survey_refs_wanted *rw = &ctx->opts.refs; + + /* + * `--all-refs` overrides and enables everything. + */ + if (rw->want_all_refs == 1) { + rw->want_branches = 1; + rw->want_tags = 1; + rw->want_remotes = 1; + rw->want_detached = 1; + rw->want_other = 1; + return; + } + + /* + * If none of the `--` were given, we assume all + * of the builtin unspecified values. + */ + if (rw->want_branches == -1 && + rw->want_tags == -1 && + rw->want_remotes == -1 && + rw->want_detached == -1 && + rw->want_other == -1) { + *rw = default_ref_options; + return; + } + + /* + * Since we only allow positive boolean values on the command + * line, we will only have true values where they specified + * a `--`. + * + * So anything that still has an unspecified value should be + * set to false. + */ + if (rw->want_branches == -1) + rw->want_branches = 0; + if (rw->want_tags == -1) + rw->want_tags = 0; + if (rw->want_remotes == -1) + rw->want_remotes = 0; + if (rw->want_detached == -1) + rw->want_detached = 0; + if (rw->want_other == -1) + rw->want_other = 0; +} + +static int survey_load_config_cb(const char *var, const char *value, + const struct config_context *cctx, void *pvoid) +{ + struct survey_context *ctx = pvoid; + + if (!strcmp(var, "survey.verbose")) { + ctx->opts.verbose = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "survey.progress")) { + ctx->opts.show_progress = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "survey.top")) { + ctx->opts.top_nr = git_config_bool(var, value); + return 0; + } + + return git_default_config(var, value, cctx, pvoid); +} + +static void survey_load_config(struct survey_context *ctx) +{ + repo_config(the_repository, survey_load_config_cb, ctx); +} + +static void do_load_refs(struct survey_context *ctx, + struct ref_array *ref_array) +{ + struct ref_filter filter = REF_FILTER_INIT; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; + + string_list_append(&sorting_options, "objectname"); + sorting = ref_sorting_options(&sorting_options); + + if (ctx->opts.refs.want_detached) + strvec_push(&ctx->refs, "HEAD"); + + if (ctx->opts.refs.want_all_refs) { + strvec_push(&ctx->refs, "refs/"); + } else { + if (ctx->opts.refs.want_branches) + strvec_push(&ctx->refs, "refs/heads/"); + if (ctx->opts.refs.want_tags) + strvec_push(&ctx->refs, "refs/tags/"); + if (ctx->opts.refs.want_remotes) + strvec_push(&ctx->refs, "refs/remotes/"); + if (ctx->opts.refs.want_other) { + strvec_push(&ctx->refs, "refs/notes/"); + strvec_push(&ctx->refs, "refs/stash/"); + } + } + + filter.name_patterns = ctx->refs.v; + filter.ignore_case = 0; + filter.match_as_path = 1; + + if (ctx->opts.show_progress) { + ctx->progress_total = 0; + ctx->progress = start_progress(ctx->repo, + _("Scanning refs..."), 0); + } + + filter_refs(ref_array, &filter, FILTER_REFS_KIND_MASK); + + if (ctx->opts.show_progress) { + ctx->progress_total = ref_array->nr; + display_progress(ctx->progress, ctx->progress_total); + } + + ref_array_sort(sorting, ref_array); + + stop_progress(&ctx->progress); + ref_filter_clear(&filter); + ref_sorting_release(sorting); +} + +/* + * The REFS phase: + * + * Load the set of requested refs and assess them for scalablity problems. + * Use that set to start a treewalk to all reachable objects and assess + * them. + * + * This data will give us insights into the repository itself (the number + * of refs, the size and shape of the DAG, the number and size of the + * objects). + * + * Theoretically, this data is independent of the on-disk representation + * (e.g. independent of packing concerns). + */ +static void survey_phase_refs(struct survey_context *ctx) +{ + trace2_region_enter("survey", "phase/refs", ctx->repo); + do_load_refs(ctx, &ctx->ref_array); + + ctx->report.refs.refs_nr = ctx->ref_array.nr; + for (int i = 0; i < ctx->ref_array.nr; i++) { + unsigned long size; + struct ref_array_item *item = ctx->ref_array.items[i]; + + switch (item->kind) { + case FILTER_REFS_TAGS: + ctx->report.refs.tags_nr++; + if (odb_read_object_info(ctx->repo->objects, + &item->objectname, + &size) == OBJ_TAG) + ctx->report.refs.tags_annotated_nr++; + break; + + case FILTER_REFS_BRANCHES: + ctx->report.refs.branches_nr++; + break; + + case FILTER_REFS_REMOTES: + ctx->report.refs.remote_refs_nr++; + break; + + case FILTER_REFS_OTHERS: + ctx->report.refs.others_nr++; + break; + + default: + ctx->report.refs.unknown_nr++; + break; + } + } + + trace2_region_leave("survey", "phase/refs", ctx->repo); +} + +static void increment_object_counts( + struct survey_report_object_summary *summary, + enum object_type type, + size_t nr) +{ + switch (type) { + case OBJ_COMMIT: + summary->commits_nr += nr; + break; + + case OBJ_TREE: + summary->trees_nr += nr; + break; + + case OBJ_BLOB: + summary->blobs_nr += nr; + break; + + case OBJ_TAG: + summary->tags_nr += nr; + break; + + default: + break; + } +} + +static void increment_totals(struct survey_context *ctx, + struct oid_array *oids, + struct survey_report_object_size_summary *summary) +{ + for (size_t i = 0; i < oids->nr; i++) { + struct object_info oi = OBJECT_INFO_INIT; + unsigned oi_flags = OBJECT_INFO_FOR_PREFETCH; + unsigned long object_length = 0; + off_t disk_sizep = 0; + enum object_type type; + + oi.typep = &type; + oi.sizep = &object_length; + oi.disk_sizep = &disk_sizep; + + if (odb_read_object_info_extended(ctx->repo->objects, + &oids->oid[i], + &oi, oi_flags) < 0) { + summary->num_missing++; + } else { + summary->nr++; + summary->disk_size += disk_sizep; + summary->inflated_size += object_length; + } + } +} + +static void increment_object_totals(struct survey_context *ctx, + struct oid_array *oids, + enum object_type type, + const char *path) +{ + struct survey_report_object_size_summary *total; + struct survey_report_object_size_summary summary = { 0 }; + + increment_totals(ctx, oids, &summary); + + switch (type) { + case OBJ_COMMIT: + total = &ctx->report.by_type[REPORT_TYPE_COMMIT]; + break; + + case OBJ_TREE: + total = &ctx->report.by_type[REPORT_TYPE_TREE]; + break; + + case OBJ_BLOB: + total = &ctx->report.by_type[REPORT_TYPE_BLOB]; + break; + + case OBJ_TAG: + total = &ctx->report.by_type[REPORT_TYPE_TAG]; + break; + + default: + BUG("No other type allowed"); + } + + total->nr += summary.nr; + total->disk_size += summary.disk_size; + total->inflated_size += summary.inflated_size; + total->num_missing += summary.num_missing; + + if (type == OBJ_TREE || type == OBJ_BLOB) { + int index = type == OBJ_TREE ? + REPORT_TYPE_TREE : REPORT_TYPE_BLOB; + struct survey_report_top_table *top; + + /* + * Temporarily store (const char *) here, but it will + * be duped if inserted and will not be freed. + */ + summary.label = (char *)path; + + top = ctx->report.top_paths_by_count; + maybe_insert_into_top_size(&top[index], &summary); + + top = ctx->report.top_paths_by_disk; + maybe_insert_into_top_size(&top[index], &summary); + + top = ctx->report.top_paths_by_inflate; + maybe_insert_into_top_size(&top[index], &summary); + } +} + +static int survey_objects_path_walk_fn(const char *path, + struct oid_array *oids, + enum object_type type, + void *data) +{ + struct survey_context *ctx = data; + + increment_object_counts(&ctx->report.reachable_objects, + type, oids->nr); + increment_object_totals(ctx, oids, type, path); + + ctx->progress_nr += oids->nr; + display_progress(ctx->progress, ctx->progress_nr); + + return 0; +} + +static void initialize_report(struct survey_context *ctx) +{ + CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT); + ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits")); + ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees")); + ctx->report.by_type[REPORT_TYPE_BLOB].label = xstrdup(_("Blobs")); + ctx->report.by_type[REPORT_TYPE_TAG].label = xstrdup(_("Tags")); + + CALLOC_ARRAY(ctx->report.top_paths_by_count, REPORT_TYPE_COUNT); + init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_TREE], + ctx->opts.top_nr, _("TOP DIRECTORIES BY COUNT"), cmp_by_nr); + init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_BLOB], + ctx->opts.top_nr, _("TOP FILES BY COUNT"), cmp_by_nr); + + CALLOC_ARRAY(ctx->report.top_paths_by_disk, REPORT_TYPE_COUNT); + init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_TREE], + ctx->opts.top_nr, _("TOP DIRECTORIES BY DISK SIZE"), cmp_by_disk_size); + init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB], + ctx->opts.top_nr, _("TOP FILES BY DISK SIZE"), cmp_by_disk_size); + + CALLOC_ARRAY(ctx->report.top_paths_by_inflate, REPORT_TYPE_COUNT); + init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE], + ctx->opts.top_nr, _("TOP DIRECTORIES BY INFLATED SIZE"), cmp_by_inflated_size); + init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB], + ctx->opts.top_nr, _("TOP FILES BY INFLATED SIZE"), cmp_by_inflated_size); +} + +static void survey_phase_objects(struct survey_context *ctx) +{ + struct rev_info revs = REV_INFO_INIT; + struct path_walk_info info = PATH_WALK_INFO_INIT; + unsigned int add_flags = 0; + + trace2_region_enter("survey", "phase/objects", ctx->repo); + + info.revs = &revs; + info.path_fn = survey_objects_path_walk_fn; + info.path_fn_data = ctx; + + initialize_report(ctx); + + repo_init_revisions(ctx->repo, &revs, ""); + revs.tag_objects = 1; + + ctx->progress_nr = 0; + ctx->progress_total = ctx->ref_array.nr; + if (ctx->opts.show_progress) + ctx->progress = start_progress(ctx->repo, + _("Preparing object walk"), + ctx->progress_total); + for (int i = 0; i < ctx->ref_array.nr; i++) { + struct ref_array_item *item = ctx->ref_array.items[i]; + add_pending_oid(&revs, NULL, &item->objectname, add_flags); + display_progress(ctx->progress, ++(ctx->progress_nr)); + } + stop_progress(&ctx->progress); + + ctx->progress_nr = 0; + ctx->progress_total = 0; + if (ctx->opts.show_progress) + ctx->progress = start_progress(ctx->repo, + _("Walking objects"), 0); + walk_objects_by_path(&info); + stop_progress(&ctx->progress); + + release_revisions(&revs); + trace2_region_leave("survey", "phase/objects", ctx->repo); +} + +int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo) +{ + static struct survey_context ctx = { + .opts = { + .verbose = 0, + .show_progress = -1, /* defaults to isatty(2) */ + .top_nr = 10, + + .refs.want_all_refs = -1, + + .refs.want_branches = -1, /* default these to undefined */ + .refs.want_tags = -1, + .refs.want_remotes = -1, + .refs.want_detached = -1, + .refs.want_other = -1, + }, + .refs = STRVEC_INIT, + }; + + static struct option survey_options[] = { + OPT__VERBOSE(&ctx.opts.verbose, N_("verbose output")), + OPT_BOOL(0, "progress", &ctx.opts.show_progress, N_("show progress")), + OPT_INTEGER('n', "top", &ctx.opts.top_nr, + N_("number of entries to include in detail tables")), + + OPT_BOOL_F(0, "all-refs", &ctx.opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), + + OPT_BOOL_F(0, "branches", &ctx.opts.refs.want_branches, N_("include branches"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "tags", &ctx.opts.refs.want_tags, N_("include tags"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "remotes", &ctx.opts.refs.want_remotes, N_("include all remotes refs"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "detached", &ctx.opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "other", &ctx.opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG), + + OPT_END(), + }; + + show_usage_with_options_if_asked(argc, argv, + survey_usage, survey_options); + + if (isatty(2)) + color_fprintf_ln(stderr, + want_color_fd(2, GIT_COLOR_AUTO) ? GIT_COLOR_YELLOW : "", + "(THIS IS EXPERIMENTAL, EXPECT THE OUTPUT FORMAT TO CHANGE!)"); + + ctx.repo = repo; + + prepare_repo_settings(ctx.repo); + survey_load_config(&ctx); + + argc = parse_options(argc, argv, prefix, survey_options, survey_usage, 0); + + if (ctx.opts.show_progress < 0) + ctx.opts.show_progress = isatty(2); + + fixup_refs_wanted(&ctx); + + survey_phase_refs(&ctx); + + survey_phase_objects(&ctx); + + survey_report_plaintext(&ctx); + + clear_survey_context(&ctx); + return 0; +} diff --git a/ci/check-whitespace.sh b/ci/check-whitespace.sh index c40804394cb079..e590ac0dfd765e 100755 --- a/ci/check-whitespace.sh +++ b/ci/check-whitespace.sh @@ -19,6 +19,7 @@ problems=() commit= commitText= commitTextmd= +committerEmail= goodParent= if ! git rev-parse --quiet --verify "${baseCommit}" @@ -27,7 +28,7 @@ then exit 1 fi -while read dash sha etc +while read dash email sha etc do case "${dash}" in "---") # Line contains commit information. @@ -40,10 +41,14 @@ do commit="${sha}" commitText="${sha} ${etc}" commitTextmd="[${sha}](${url}/commit/${sha}) ${etc}" + committerEmail="${email}" ;; "") ;; *) # Line contains whitespace error information for current commit. + # Quod licet Iovi non licet bovi + test gitster@pobox.com != "$committerEmail" || break + if test -n "${goodParent}" then problems+=("1) --- ${commitTextmd}") @@ -64,7 +69,7 @@ do echo "${dash} ${sha} ${etc}" ;; esac -done <<< "$(git log --check --pretty=format:"---% h% s" "${baseCommit}"..)" +done <<< "$(git log --check --pretty=format:"---% ce% h% s" "${baseCommit}"..)" if test ${#problems[*]} -gt 0 then diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 10c3530d1aacdd..02af0491ee2fa2 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -119,11 +119,12 @@ macos-*) # brew install gnu-time brew link --force gettext - mkdir -p "$CUSTOM_PATH" - wget -q "$P4WHENCE/bin.macosx12arm64/helix-core-server.tgz" && - tar -xf helix-core-server.tgz -C "$CUSTOM_PATH" p4 p4d && - sudo xattr -d com.apple.quarantine "$CUSTOM_PATH/p4" "$CUSTOM_PATH/p4d" 2>/dev/null || true - rm helix-core-server.tgz + # Uncomment this block if you want to run `git p4` tests: + # mkdir -p "$CUSTOM_PATH" + # wget -q "$P4WHENCE/bin.macosx12arm64/helix-core-server.tgz" && + # tar -xf helix-core-server.tgz -C "$CUSTOM_PATH" p4 p4d && + # sudo xattr -d com.apple.quarantine "$CUSTOM_PATH/p4" "$CUSTOM_PATH/p4d" 2>/dev/null || true + # rm helix-core-server.tgz case "$jobname" in osx-meson) diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 1d9a0a736d255b..7133cd901a9ea2 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -72,5 +72,9 @@ case "$jobname" in ;; esac +case " $MAKE_TARGETS " in +*" all "*) make -C contrib/subtree test;; +esac + check_unignored_build_artifacts save_good_tree diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index ff948e397fcb70..f84190e7b73180 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -15,4 +15,7 @@ if [ "$1" == "0" ] ; then group "Run unit tests" make --quiet -C t unit-tests-test-tool fi +# Run the git subtree tests only if main tests succeeded +test 0 != "$1" || make -C contrib/subtree test + check_unignored_build_artifacts diff --git a/command-list.txt b/command-list.txt index 21b802c42026b3..444e8e7695f065 100644 --- a/command-list.txt +++ b/command-list.txt @@ -192,6 +192,7 @@ git-stash mainporcelain git-status mainporcelain info git-stripspace purehelpers git-submodule mainporcelain +git-survey mainporcelain git-svn foreignscminterface git-switch mainporcelain history git-symbolic-ref plumbingmanipulators diff --git a/common-exit.c b/common-exit.c index 1aaa538be3ed67..609f32abed8b53 100644 --- a/common-exit.c +++ b/common-exit.c @@ -11,6 +11,13 @@ static void check_bug_if_BUG(void) /* We wrap exit() to call common_exit() in git-compat-util.h */ int common_exit(const char *file, int line, int code) { + /* + * Windows Filtering Platform driver provided by the security software + * may change buffer type of stdout from _IONBF to _IOFBF. + * It will no output without fflush manually. + */ + fflush(stdout); + /* * For non-POSIX systems: Take the lowest 8 bits of the "code" * to e.g. turn -1 into 255. On a POSIX system this is diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 2aa8c219acee4d..4b53360d194105 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -34,7 +34,7 @@ struct fsm_health_data struct wt_moved { - wchar_t wpath[MAX_PATH + 1]; + wchar_t wpath[MAX_LONG_PATH + 1]; BY_HANDLE_FILE_INFORMATION bhfi; } wt_moved; }; @@ -143,8 +143,8 @@ static int has_worktree_moved(struct fsmonitor_daemon_state *state, return 0; case CTX_INIT: - if (xutftowcs_path(data->wt_moved.wpath, - state->path_worktree_watch.buf) < 0) { + if (xutftowcs_long_path(data->wt_moved.wpath, + state->path_worktree_watch.buf) < 0) { error(_("could not convert to wide characters: '%s'"), state->path_worktree_watch.buf); return -1; diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 9a6efc9bea340b..afcc172750af10 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -28,7 +28,7 @@ struct one_watch DWORD count; struct strbuf path; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; DWORD wpath_longname_len; HANDLE hDir; @@ -131,8 +131,8 @@ static int normalize_path_in_utf8(wchar_t *wpath, DWORD wpath_len, */ static void check_for_shortnames(struct one_watch *watch) { - wchar_t buf_in[MAX_PATH + 1]; - wchar_t buf_out[MAX_PATH + 1]; + wchar_t buf_in[MAX_LONG_PATH + 1]; + wchar_t buf_out[MAX_LONG_PATH + 1]; wchar_t *last; wchar_t *p; @@ -197,8 +197,8 @@ static enum get_relative_result get_relative_longname( const wchar_t *wpath, DWORD wpath_len, wchar_t *wpath_longname, size_t bufsize_wpath_longname) { - wchar_t buf_in[2 * MAX_PATH + 1]; - wchar_t buf_out[MAX_PATH + 1]; + wchar_t buf_in[2 * MAX_LONG_PATH + 1]; + wchar_t buf_out[MAX_LONG_PATH + 1]; DWORD root_len; DWORD out_len; @@ -298,10 +298,10 @@ static struct one_watch *create_watch(const char *path) FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; HANDLE hDir; DWORD len_longname; - wchar_t wpath[MAX_PATH + 1]; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath[MAX_LONG_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; - if (xutftowcs_path(wpath, path) < 0) { + if (xutftowcs_long_path(wpath, path) < 0) { error(_("could not convert to wide characters: '%s'"), path); return NULL; } @@ -545,7 +545,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) struct string_list cookie_list = STRING_LIST_INIT_DUP; struct fsmonitor_batch *batch = NULL; const char *p = watch->buffer; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; /* * If the kernel gets more events than will fit in the kernel diff --git a/compat/fsmonitor/fsm-path-utils-win32.c b/compat/fsmonitor/fsm-path-utils-win32.c index f4f9cc1f336720..c6eb065bde48b4 100644 --- a/compat/fsmonitor/fsm-path-utils-win32.c +++ b/compat/fsmonitor/fsm-path-utils-win32.c @@ -69,8 +69,8 @@ static int check_remote_protocol(wchar_t *wpath) */ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info) { - wchar_t wpath[MAX_PATH]; - wchar_t wfullpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; + wchar_t wfullpath[MAX_LONG_PATH]; size_t wlen; UINT driveType; @@ -78,7 +78,7 @@ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info) * Do everything in wide chars because the drive letter might be * a multi-byte sequence. See win32_has_dos_drive_prefix(). */ - if (xutftowcs_path(wpath, path) < 0) { + if (xutftowcs_long_path(wpath, path) < 0) { return -1; } @@ -97,7 +97,7 @@ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info) * slashes to backslashes. This is essential to get GetDriveTypeW() * correctly handle some UNC "\\server\share\..." paths. */ - if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) { + if (!GetFullPathNameW(wpath, MAX_LONG_PATH, wfullpath, NULL)) { return -1; } diff --git a/compat/lazyload-curl.c b/compat/lazyload-curl.c new file mode 100644 index 00000000000000..a6a3f7e3a7aeaa --- /dev/null +++ b/compat/lazyload-curl.c @@ -0,0 +1,428 @@ +#include "../git-compat-util.h" +#include "../git-curl-compat.h" +#ifndef WIN32 +#include +#endif + +/* + * The ABI version of libcurl is encoded in its shared libraries' file names. + * This ABI version has not changed since October 2006 and is unlikely to be + * changed in the future. See https://curl.se/libcurl/abi.html for details. + */ +#define LIBCURL_ABI_VERSION "4" + +typedef void (*func_t)(void); + +#ifndef WIN32 +#ifdef __APPLE__ +#define LIBCURL_FILE_NAME(base) base "." LIBCURL_ABI_VERSION ".dylib" +#else +#define LIBCURL_FILE_NAME(base) base ".so." LIBCURL_ABI_VERSION +#endif + +static void *load_library(const char *name) +{ + return dlopen(name, RTLD_LAZY); +} + +static func_t load_function(void *handle, const char *name) +{ + /* + * Casting the return value of `dlsym()` to a function pointer is + * explicitly allowed in recent POSIX standards, but GCC complains + * about this in pedantic mode nevertheless. For more about this issue, + * see https://stackoverflow.com/q/31526876/1860823 and + * http://stackoverflow.com/a/36385690/1905491. + */ + func_t f; + *(void **)&f = dlsym(handle, name); + return f; +} +#else +#define LIBCURL_FILE_NAME(base) base "-" LIBCURL_ABI_VERSION ".dll" + +static void *load_library(const char *name) +{ + size_t name_size = strlen(name) + 1; + const char *path = getenv("PATH"); + char dll_path[MAX_PATH]; + + while (path && *path) { + const char *sep = strchrnul(path, ';'); + size_t len = sep - path; + + if (len && len + name_size < sizeof(dll_path)) { + memcpy(dll_path, path, len); + dll_path[len] = '/'; + memcpy(dll_path + len + 1, name, name_size); + + if (!access(dll_path, R_OK)) { + wchar_t wpath[MAX_PATH]; + int wlen = MultiByteToWideChar(CP_UTF8, 0, dll_path, -1, wpath, ARRAY_SIZE(wpath)); + void *res = wlen ? (void *)LoadLibraryExW(wpath, NULL, 0) : NULL; + if (!res) { + DWORD err = GetLastError(); + char buf[1024]; + + if (!FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_ARGUMENT_ARRAY | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, LANG_NEUTRAL, + buf, sizeof(buf) - 1, NULL)) + xsnprintf(buf, sizeof(buf), "last error: %ld", err); + error("LoadLibraryExW() failed with: %s", buf); + } + return res; + } + } + + path = *sep ? sep + 1 : NULL; + } + + return NULL; +} + +static func_t load_function(void *handle, const char *name) +{ + return (func_t)GetProcAddress((HANDLE)handle, name); +} +#endif + +typedef struct curl_version_info_data *(*curl_version_info_type)(CURLversion version); +static curl_version_info_type curl_version_info_func; + +typedef char *(*curl_easy_escape_type)(CURL *handle, const char *string, int length); +static curl_easy_escape_type curl_easy_escape_func; + +typedef void (*curl_free_type)(void *p); +static curl_free_type curl_free_func; + +typedef CURLcode (*curl_global_init_type)(long flags); +static curl_global_init_type curl_global_init_func; + +typedef CURLsslset (*curl_global_sslset_type)(curl_sslbackend id, const char *name, const curl_ssl_backend ***avail); +static curl_global_sslset_type curl_global_sslset_func; + +typedef void (*curl_global_cleanup_type)(void); +static curl_global_cleanup_type curl_global_cleanup_func; + +typedef CURLcode (*curl_global_trace_type)(const char *config); +static curl_global_trace_type curl_global_trace_func; + +typedef struct curl_slist *(*curl_slist_append_type)(struct curl_slist *list, const char *data); +static curl_slist_append_type curl_slist_append_func; + +typedef void (*curl_slist_free_all_type)(struct curl_slist *list); +static curl_slist_free_all_type curl_slist_free_all_func; + +typedef const char *(*curl_easy_strerror_type)(CURLcode error); +static curl_easy_strerror_type curl_easy_strerror_func; + +typedef CURLM *(*curl_multi_init_type)(void); +static curl_multi_init_type curl_multi_init_func; + +typedef CURLMcode (*curl_multi_add_handle_type)(CURLM *multi_handle, CURL *curl_handle); +static curl_multi_add_handle_type curl_multi_add_handle_func; + +typedef CURLMcode (*curl_multi_remove_handle_type)(CURLM *multi_handle, CURL *curl_handle); +static curl_multi_remove_handle_type curl_multi_remove_handle_func; + +typedef CURLMcode (*curl_multi_fdset_type)(CURLM *multi_handle, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *exc_fd_set, int *max_fd); +static curl_multi_fdset_type curl_multi_fdset_func; + +typedef CURLMcode (*curl_multi_perform_type)(CURLM *multi_handle, int *running_handles); +static curl_multi_perform_type curl_multi_perform_func; + +typedef CURLMcode (*curl_multi_cleanup_type)(CURLM *multi_handle); +static curl_multi_cleanup_type curl_multi_cleanup_func; + +typedef CURLMsg *(*curl_multi_info_read_type)(CURLM *multi_handle, int *msgs_in_queue); +static curl_multi_info_read_type curl_multi_info_read_func; + +typedef const char *(*curl_multi_strerror_type)(CURLMcode error); +static curl_multi_strerror_type curl_multi_strerror_func; + +typedef CURLMcode (*curl_multi_timeout_type)(CURLM *multi_handle, long *milliseconds); +static curl_multi_timeout_type curl_multi_timeout_func; + +typedef CURL *(*curl_easy_init_type)(void); +static curl_easy_init_type curl_easy_init_func; + +typedef CURLcode (*curl_easy_perform_type)(CURL *curl); +static curl_easy_perform_type curl_easy_perform_func; + +typedef void (*curl_easy_cleanup_type)(CURL *curl); +static curl_easy_cleanup_type curl_easy_cleanup_func; + +typedef CURL *(*curl_easy_duphandle_type)(CURL *curl); +static curl_easy_duphandle_type curl_easy_duphandle_func; + +typedef CURLcode (*curl_easy_getinfo_long_type)(CURL *curl, CURLINFO info, long *value); +static curl_easy_getinfo_long_type curl_easy_getinfo_long_func; + +typedef CURLcode (*curl_easy_getinfo_pointer_type)(CURL *curl, CURLINFO info, void **value); +static curl_easy_getinfo_pointer_type curl_easy_getinfo_pointer_func; + +typedef CURLcode (*curl_easy_getinfo_off_t_type)(CURL *curl, CURLINFO info, curl_off_t *value); +static curl_easy_getinfo_off_t_type curl_easy_getinfo_off_t_func; + +typedef CURLcode (*curl_easy_setopt_long_type)(CURL *curl, CURLoption opt, long value); +static curl_easy_setopt_long_type curl_easy_setopt_long_func; + +typedef CURLcode (*curl_easy_setopt_pointer_type)(CURL *curl, CURLoption opt, void *value); +static curl_easy_setopt_pointer_type curl_easy_setopt_pointer_func; + +typedef CURLcode (*curl_easy_setopt_off_t_type)(CURL *curl, CURLoption opt, curl_off_t value); +static curl_easy_setopt_off_t_type curl_easy_setopt_off_t_func; + +static char ssl_backend[64]; + +static void lazy_load_curl(void) +{ + static int initialized; + void *libcurl = NULL; + func_t curl_easy_getinfo_func, curl_easy_setopt_func; + + if (initialized) + return; + + initialized = 1; + if (ssl_backend[0]) { + char dll_name[64 + 16]; + snprintf(dll_name, sizeof(dll_name) - 1, + LIBCURL_FILE_NAME("libcurl-%s"), ssl_backend); + libcurl = load_library(dll_name); + } + if (!libcurl) + libcurl = load_library(LIBCURL_FILE_NAME("libcurl")); + if (!libcurl) + die("failed to load library '%s'", LIBCURL_FILE_NAME("libcurl")); + + curl_version_info_func = (curl_version_info_type)load_function(libcurl, "curl_version_info"); + curl_easy_escape_func = (curl_easy_escape_type)load_function(libcurl, "curl_easy_escape"); + curl_free_func = (curl_free_type)load_function(libcurl, "curl_free"); + curl_global_init_func = (curl_global_init_type)load_function(libcurl, "curl_global_init"); + curl_global_sslset_func = (curl_global_sslset_type)load_function(libcurl, "curl_global_sslset"); + curl_global_cleanup_func = (curl_global_cleanup_type)load_function(libcurl, "curl_global_cleanup"); + curl_global_trace_func = (curl_global_trace_type)load_function(libcurl, "curl_global_trace"); + curl_slist_append_func = (curl_slist_append_type)load_function(libcurl, "curl_slist_append"); + curl_slist_free_all_func = (curl_slist_free_all_type)load_function(libcurl, "curl_slist_free_all"); + curl_easy_strerror_func = (curl_easy_strerror_type)load_function(libcurl, "curl_easy_strerror"); + curl_multi_init_func = (curl_multi_init_type)load_function(libcurl, "curl_multi_init"); + curl_multi_add_handle_func = (curl_multi_add_handle_type)load_function(libcurl, "curl_multi_add_handle"); + curl_multi_remove_handle_func = (curl_multi_remove_handle_type)load_function(libcurl, "curl_multi_remove_handle"); + curl_multi_fdset_func = (curl_multi_fdset_type)load_function(libcurl, "curl_multi_fdset"); + curl_multi_perform_func = (curl_multi_perform_type)load_function(libcurl, "curl_multi_perform"); + curl_multi_cleanup_func = (curl_multi_cleanup_type)load_function(libcurl, "curl_multi_cleanup"); + curl_multi_info_read_func = (curl_multi_info_read_type)load_function(libcurl, "curl_multi_info_read"); + curl_multi_strerror_func = (curl_multi_strerror_type)load_function(libcurl, "curl_multi_strerror"); + curl_multi_timeout_func = (curl_multi_timeout_type)load_function(libcurl, "curl_multi_timeout"); + curl_easy_init_func = (curl_easy_init_type)load_function(libcurl, "curl_easy_init"); + curl_easy_perform_func = (curl_easy_perform_type)load_function(libcurl, "curl_easy_perform"); + curl_easy_cleanup_func = (curl_easy_cleanup_type)load_function(libcurl, "curl_easy_cleanup"); + curl_easy_duphandle_func = (curl_easy_duphandle_type)load_function(libcurl, "curl_easy_duphandle"); + + curl_easy_getinfo_func = load_function(libcurl, "curl_easy_getinfo"); + curl_easy_getinfo_long_func = (curl_easy_getinfo_long_type)curl_easy_getinfo_func; + curl_easy_getinfo_pointer_func = (curl_easy_getinfo_pointer_type)curl_easy_getinfo_func; + curl_easy_getinfo_off_t_func = (curl_easy_getinfo_off_t_type)curl_easy_getinfo_func; + + curl_easy_setopt_func = load_function(libcurl, "curl_easy_setopt"); + curl_easy_setopt_long_func = (curl_easy_setopt_long_type)curl_easy_setopt_func; + curl_easy_setopt_pointer_func = (curl_easy_setopt_pointer_type)curl_easy_setopt_func; + curl_easy_setopt_off_t_func = (curl_easy_setopt_off_t_type)curl_easy_setopt_func; +} + +struct curl_version_info_data *curl_version_info(CURLversion version) +{ + lazy_load_curl(); + return curl_version_info_func(version); +} + +char *curl_easy_escape(CURL *handle, const char *string, int length) +{ + lazy_load_curl(); + return curl_easy_escape_func(handle, string, length); +} + +void curl_free(void *p) +{ + lazy_load_curl(); + curl_free_func(p); +} + +CURLcode curl_global_init(long flags) +{ + lazy_load_curl(); + return curl_global_init_func(flags); +} + +CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, const curl_ssl_backend ***avail) +{ + if (name && strlen(name) < sizeof(ssl_backend)) + strlcpy(ssl_backend, name, sizeof(ssl_backend)); + + lazy_load_curl(); + return curl_global_sslset_func(id, name, avail); +} + +void curl_global_cleanup(void) +{ + lazy_load_curl(); + curl_global_cleanup_func(); +} + +CURLcode curl_global_trace(const char *config) +{ + lazy_load_curl(); + return curl_global_trace_func(config); +} + +struct curl_slist *curl_slist_append(struct curl_slist *list, const char *data) +{ + lazy_load_curl(); + return curl_slist_append_func(list, data); +} + +void curl_slist_free_all(struct curl_slist *list) +{ + lazy_load_curl(); + curl_slist_free_all_func(list); +} + +const char *curl_easy_strerror(CURLcode error) +{ + lazy_load_curl(); + return curl_easy_strerror_func(error); +} + +CURLM *curl_multi_init(void) +{ + lazy_load_curl(); + return curl_multi_init_func(); +} + +CURLMcode curl_multi_add_handle(CURLM *multi_handle, CURL *curl_handle) +{ + lazy_load_curl(); + return curl_multi_add_handle_func(multi_handle, curl_handle); +} + +CURLMcode curl_multi_remove_handle(CURLM *multi_handle, CURL *curl_handle) +{ + lazy_load_curl(); + return curl_multi_remove_handle_func(multi_handle, curl_handle); +} + +CURLMcode curl_multi_fdset(CURLM *multi_handle, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *exc_fd_set, int *max_fd) +{ + lazy_load_curl(); + return curl_multi_fdset_func(multi_handle, read_fd_set, write_fd_set, exc_fd_set, max_fd); +} + +CURLMcode curl_multi_perform(CURLM *multi_handle, int *running_handles) +{ + lazy_load_curl(); + return curl_multi_perform_func(multi_handle, running_handles); +} + +CURLMcode curl_multi_cleanup(CURLM *multi_handle) +{ + lazy_load_curl(); + return curl_multi_cleanup_func(multi_handle); +} + +CURLMsg *curl_multi_info_read(CURLM *multi_handle, int *msgs_in_queue) +{ + lazy_load_curl(); + return curl_multi_info_read_func(multi_handle, msgs_in_queue); +} + +const char *curl_multi_strerror(CURLMcode error) +{ + lazy_load_curl(); + return curl_multi_strerror_func(error); +} + +CURLMcode curl_multi_timeout(CURLM *multi_handle, long *milliseconds) +{ + lazy_load_curl(); + return curl_multi_timeout_func(multi_handle, milliseconds); +} + +CURL *curl_easy_init(void) +{ + lazy_load_curl(); + return curl_easy_init_func(); +} + +CURLcode curl_easy_perform(CURL *curl) +{ + lazy_load_curl(); + return curl_easy_perform_func(curl); +} + +void curl_easy_cleanup(CURL *curl) +{ + lazy_load_curl(); + curl_easy_cleanup_func(curl); +} + +CURL *curl_easy_duphandle(CURL *curl) +{ + lazy_load_curl(); + return curl_easy_duphandle_func(curl); +} + +#ifndef CURL_IGNORE_DEPRECATION +#define CURL_IGNORE_DEPRECATION(x) x +#endif + +#ifndef CURLOPTTYPE_BLOB +#define CURLOPTTYPE_BLOB 40000 +#endif + +#undef curl_easy_getinfo +CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...) +{ + va_list ap; + CURLcode res; + + va_start(ap, info); + lazy_load_curl(); + CURL_IGNORE_DEPRECATION( + if (info >= CURLINFO_LONG && info < CURLINFO_DOUBLE) + res = curl_easy_getinfo_long_func(curl, info, va_arg(ap, long *)); + else if ((info >= CURLINFO_STRING && info < CURLINFO_LONG) || + (info >= CURLINFO_SLIST && info < CURLINFO_SOCKET)) + res = curl_easy_getinfo_pointer_func(curl, info, va_arg(ap, void **)); + else if (info >= CURLINFO_OFF_T) + res = curl_easy_getinfo_off_t_func(curl, info, va_arg(ap, curl_off_t *)); + else + die("%s:%d: TODO (info: %d)!", __FILE__, __LINE__, info); + ) + va_end(ap); + return res; +} + +#undef curl_easy_setopt +CURLcode curl_easy_setopt(CURL *curl, CURLoption opt, ...) +{ + va_list ap; + CURLcode res; + + va_start(ap, opt); + lazy_load_curl(); + CURL_IGNORE_DEPRECATION( + if (opt >= CURLOPTTYPE_LONG && opt < CURLOPTTYPE_OBJECTPOINT) + res = curl_easy_setopt_long_func(curl, opt, va_arg(ap, long)); + else if (opt >= CURLOPTTYPE_OBJECTPOINT && opt < CURLOPTTYPE_OFF_T) + res = curl_easy_setopt_pointer_func(curl, opt, va_arg(ap, void *)); + else if (opt >= CURLOPTTYPE_OFF_T && opt < CURLOPTTYPE_BLOB) + res = curl_easy_setopt_off_t_func(curl, opt, va_arg(ap, curl_off_t)); + else + die("%s:%d: TODO (opt: %d)!", __FILE__, __LINE__, opt); + ) + va_end(ap); + return res; +} diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h index 2d989fd762474e..9158f89d89d239 100644 --- a/compat/mingw-posix.h +++ b/compat/mingw-posix.h @@ -193,8 +193,10 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); -int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); +struct index_state; +int mingw_create_symlink(struct index_state *index, const char *target, const char *link); +#define create_symlink mingw_create_symlink /* * replacements of existing functions @@ -288,6 +290,11 @@ int mingw_socket(int domain, int type, int protocol); int mingw_connect(int sockfd, struct sockaddr *sa, size_t sz); #define connect mingw_connect +char *mingw_strerror(int errnum); +#ifndef _UCRT +#define strerror mingw_strerror +#endif + int mingw_bind(int sockfd, struct sockaddr *sa, size_t sz); #define bind mingw_bind @@ -333,6 +340,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows, * including our own struct stat with 64 bit st_size and nanosecond-precision @@ -349,6 +367,13 @@ struct timespec { #endif #endif +static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) +{ + long long hnsec = filetime_to_hnsec(ft); + ts->tv_sec = (time_t)(hnsec / 10000000); + ts->tv_nsec = (hnsec % 10000000) * 100; +} + struct mingw_stat { _dev_t st_dev; _ino_t st_ino; @@ -381,7 +406,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); int mingw_utime(const char *file_name, const struct utimbuf *times); diff --git a/compat/mingw.c b/compat/mingw.c index aa7525f419cb64..940243e0a3b67c 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -4,17 +4,24 @@ #include "git-compat-util.h" #include "abspath.h" #include "alloc.h" +#include "attr.h" #include "config.h" #include "dir.h" #include "environment.h" #include "gettext.h" +#include "repository.h" #include "run-command.h" #include "strbuf.h" +#include "string-list.h" #include "symlinks.h" #include "trace2.h" #include "win32.h" +#include "win32/exit-process.h" +#include "win32/fscache.h" #include "win32/lazyload.h" +#include "win32/wsl.h" #include "wrapper.h" +#include "write-or-die.h" #include #include #include @@ -272,6 +279,28 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; +int core_fscache; + +int are_long_paths_enabled(void) +{ + /* default to `false` during initialization */ + static const int fallback = 0; + + static int enabled = -1; + + if (enabled < 0) { + /* avoid infinite recursion */ + if (!the_repository) + return fallback; + + if (the_repository->config && + the_repository->config->hash_initialized && + repo_config_get_bool(the_repository, "core.longpaths", &enabled) < 0) + enabled = 0; + } + + return enabled < 0 ? fallback : enabled; +} int mingw_core_config(const char *var, const char *value, const struct config_context *ctx UNUSED, @@ -285,6 +314,11 @@ int mingw_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { if (!value) return config_error_nonbool(var); @@ -348,9 +382,32 @@ process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) { HANDLE hnd; BY_HANDLE_FILE_INFORMATION fdata; - wchar_t relative[MAX_PATH]; + wchar_t relative[MAX_LONG_PATH]; const wchar_t *rel; + /* + * Do not follow symlinks to network shares, to avoid NTLM credential + * leak from crafted repositories (e.g. \\attacker-server\share). + * Since paths come in all kind of enterprising shapes and forms (in + * addition to the canonical `\\host\share` form, there's also + * `\??\UNC\host\share`, `\GLOBAL??\UNC\host\share` and also + * `\Device\Mup\host\share`, just to name a few), we simply avoid + * following every symlink target that starts with a slash. + * + * This also catches drive-less absolute paths, of course. These are + * uncommon in practice (and also fragile because they are relative to + * the current working directory's drive). The only "harm" this does + * is that it now requires users to specify via the Git attributes if + * they have such an uncommon symbolic link and need it to be a + * directory type link. + */ + if (is_wdir_sep(wtarget[0])) { + warning("created file symlink '%ls' pointing to '%ls';\n" + "set the `symlink` gitattribute to `dir` if a " + "directory symlink is required", wlink, wtarget); + return PHANTOM_SYMLINK_DONE; + } + /* check that wlink is still a file symlink */ if ((GetFileAttributesW(wlink) & (FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_DIRECTORY)) @@ -424,6 +481,54 @@ static void process_phantom_symlinks(void) LeaveCriticalSection(&phantom_symlinks_cs); } +static int create_phantom_symlink(wchar_t *wtarget, wchar_t *wlink) +{ + int len; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } + return 0; +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -492,8 +597,8 @@ int mingw_unlink(const char *pathname, int handle_in_use_error) { static int use_legacy_delete = -1; int tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; if (use_legacy_delete < 0) @@ -528,7 +633,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -549,7 +654,7 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int tries = 0; - wchar_t wpathname[MAX_PATH]; + wchar_t wpathname[MAX_LONG_PATH]; struct stat st; /* @@ -571,7 +676,7 @@ int mingw_rmdir(const char *pathname) return -1; } - if (xutftowcs_path(wpathname, pathname) < 0) + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; do { @@ -640,15 +745,18 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode UNUSED) { int ret; - wchar_t wpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; if (!is_valid_win32_path(path, 0)) { errno = EINVAL; return -1; } - if (xutftowcs_path(wpath, path) < 0) + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + are_long_paths_enabled()) < 0) return -1; + ret = _wmkdir(wpath); if (!ret) process_phantom_symlinks(); @@ -809,11 +917,12 @@ static int is_local_named_pipe_path(const char *filename) int mingw_open (const char *filename, int oflags, ...) { + static int append_atomically = -1; typedef int (*open_fn_t)(wchar_t const *wfilename, int oflags, ...); va_list args; unsigned mode; int fd, create = (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL); - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; open_fn_t open_fn; WIN32_FILE_ATTRIBUTE_DATA fdata; @@ -828,7 +937,16 @@ int mingw_open (const char *filename, int oflags, ...) return -1; } - if ((oflags & O_APPEND) && !is_local_named_pipe_path(filename)) + /* + * Only set append_atomically to default value(1) when repo is initialized + * and fail to get config value + */ + if (append_atomically < 0 && the_repository && the_repository->commondir && + repo_config_get_bool(the_repository, "windows.appendatomically", &append_atomically)) + append_atomically = 1; + + if (append_atomically && (oflags & O_APPEND) && + !is_local_named_pipe_path(filename)) open_fn = mingw_open_append; else if (!(oflags & ~(O_ACCMODE | O_NOINHERIT))) open_fn = mingw_open_existing; @@ -837,7 +955,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); - else if (xutftowcs_path(wfilename, filename) < 0) + else if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* @@ -870,6 +988,11 @@ int mingw_open (const char *filename, int oflags, ...) if (fd < 0 && create && GetLastError() == ERROR_ACCESS_DENIED && INIT_PROC_ADDR(RtlGetLastNtStatus) && RtlGetLastNtStatus() == STATUS_DELETE_PENDING) errno = EEXIST; + else if ((oflags & O_CREAT) && fd >= 0 && are_wsl_compatible_mode_bits_enabled()) { + _mode_t wsl_mode = S_IFREG | (mode&0777); + set_wsl_mode_bits_by_handle((HANDLE)_get_osfhandle(fd), wsl_mode); + } + if (fd < 0 && (oflags & O_ACCMODE) != O_RDONLY && errno == EACCES) { DWORD attrs = GetFileAttributesW(wfilename); if (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY)) @@ -923,14 +1046,14 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); else if (!is_valid_win32_path(filename, 1)) { int create = otype && strchr(otype, 'w'); errno = create ? EINVAL : ENOENT; return NULL; - } else if (xutftowcs_path(wfilename, filename) < 0) + } else if (xutftowcs_long_path(wfilename, filename) < 0) return NULL; if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) @@ -952,14 +1075,14 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); else if (!is_valid_win32_path(filename, 1)) { int create = otype && strchr(otype, 'w'); errno = create ? EINVAL : ENOENT; return NULL; - } else if (xutftowcs_path(wfilename, filename) < 0) + } else if (xutftowcs_long_path(wfilename, filename) < 0) return NULL; if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) @@ -1002,14 +1125,33 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) { ssize_t result = write(fd, buf, len); - if (result < 0 && (errno == EINVAL || errno == ENOSPC) && buf) { + if (result < 0 && (errno == EINVAL || errno == EBADF || errno == ENOSPC) && buf) { int orig = errno; /* check if fd is a pipe */ HANDLE h = (HANDLE) _get_osfhandle(fd); - if (GetFileType(h) != FILE_TYPE_PIPE) + if (GetFileType(h) != FILE_TYPE_PIPE) { + if (orig == EINVAL) { + wchar_t path[MAX_LONG_PATH]; + DWORD ret = GetFinalPathNameByHandleW(h, path, + ARRAY_SIZE(path), 0); + UINT drive_type = ret > 0 && ret < ARRAY_SIZE(path) ? + GetDriveTypeW(path) : DRIVE_UNKNOWN; + + /* + * The default atomic append causes such an error on + * network file systems, in such a case, it should be + * turned off via config. + * + * `drive_type` of UNC path: DRIVE_NO_ROOT_DIR + */ + if (DRIVE_NO_ROOT_DIR == drive_type || DRIVE_REMOTE == drive_type) + warning("invalid write operation detected; you may try:\n" + "\n\tgit config windows.appendAtomically false"); + } + errno = orig; - else if (orig == EINVAL) + } else if (orig == EINVAL || errno == EBADF) errno = EPIPE; else { DWORD buf_size; @@ -1027,20 +1169,23 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; if (!strcmp("nul", filename) || !strcmp("/dev/null", filename)) return 0; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { - wchar_t wdirname[MAX_PATH]; - - if (xutftowcs_path(wdirname, dirname) < 0) + int result; + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; if (has_symlinks) { @@ -1059,35 +1204,19 @@ int mingw_chdir(const char *dirname) CloseHandle(hnd); } - return _wchdir(normalize_ntpath(wdirname)); + result = _wchdir(normalize_ntpath(wdirname)); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) -{ - long long hnsec = filetime_to_hnsec(ft); - ts->tv_sec = (time_t)(hnsec / 10000000); - ts->tv_nsec = (hnsec % 10000000) * 100; -} - /** * Verifies that safe_create_leading_directories() would succeed. */ @@ -1217,8 +1346,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) WIN32_FILE_ATTRIBUTE_DATA fdata; DWORD reparse_tag = 0; int link_len = 0; - wchar_t wfilename[MAX_PATH]; - int wlen = xutftowcs_path(wfilename, file_name); + wchar_t wfilename[MAX_LONG_PATH]; + int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) return -1; @@ -1233,7 +1362,7 @@ int mingw_lstat(const char *file_name, struct stat *buf) if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { /* for reparse points, get the link tag and length */ if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - char tmpbuf[MAX_PATH]; + char tmpbuf[MAX_LONG_PATH]; if (read_reparse_point(wfilename, FALSE, tmpbuf, &link_len, &reparse_tag) < 0) @@ -1244,13 +1373,18 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_uid = 0; buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, - reparse_tag); + reparse_tag, file_name); buf->st_size = S_ISLNK(buf->st_mode) ? link_len : fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); + if (S_ISREG(buf->st_mode) && + are_wsl_compatible_mode_bits_enabled()) { + copy_wsl_mode_bits_from_disk(wfilename, -1, + &buf->st_mode); + } return 0; } @@ -1280,6 +1414,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) return -1; } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) { BY_HANDLE_FILE_INFORMATION fdata; @@ -1293,24 +1429,26 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0, NULL); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); + if (are_wsl_compatible_mode_bits_enabled()) + get_wsl_mode_bits_by_handle(hnd, &buf->st_mode); return 0; } int mingw_stat(const char *file_name, struct stat *buf) { - wchar_t wfile_name[MAX_PATH]; + wchar_t wfile_name[MAX_LONG_PATH]; HANDLE hnd; int result; /* open the file and let Windows resolve the links */ - if (xutftowcs_path(wfile_name, file_name) < 0) + if (xutftowcs_long_path(wfile_name, file_name) < 0) return -1; hnd = CreateFileW(wfile_name, 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, @@ -1378,10 +1516,10 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; HANDLE osfilehandle; - if (xutftowcs_path(wfilename, file_name) < 0) + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -1520,6 +1658,82 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) } #endif +char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path) +{ + wchar_t wpath[MAX_PATH]; + HANDLE h; + DWORD ret; + int len; + const char *last_component = NULL; + char *append = NULL; + + if (xutftowcs_path(wpath, path) < 0) + return NULL; + + h = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + /* + * strbuf_realpath() allows the last path component to not exist. If + * that is the case, now it's time to try without last component. + */ + if (h == INVALID_HANDLE_VALUE && + GetLastError() == ERROR_FILE_NOT_FOUND) { + /* cut last component off of `wpath` */ + wchar_t *p = wpath + wcslen(wpath); + + while (p != wpath) + if (*(--p) == L'/' || *p == L'\\') + break; /* found start of last component */ + + if (p != wpath && (last_component = find_last_dir_sep(path))) { + append = xstrdup(last_component + 1); /* skip directory separator */ + /* + * Do not strip the trailing slash at the drive root, otherwise + * the path would be e.g. `C:` (which resolves to the + * _current_ directory on that drive). + */ + if (p[-1] == L':') + p[1] = L'\0'; + else + *p = L'\0'; + h = CreateFileW(wpath, 0, FILE_SHARE_READ | + FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); + } + } + + if (h == INVALID_HANDLE_VALUE) { +realpath_failed: + FREE_AND_NULL(append); + return NULL; + } + + ret = GetFinalPathNameByHandleW(h, wpath, ARRAY_SIZE(wpath), 0); + CloseHandle(h); + if (!ret || ret >= ARRAY_SIZE(wpath)) + goto realpath_failed; + + len = wcslen(wpath) * 3; + strbuf_grow(resolved, len); + len = xwcstoutf(resolved->buf, normalize_ntpath(wpath), len); + if (len < 0) + goto realpath_failed; + resolved->len = len; + + if (append) { + /* Use forward-slash, like `normalize_ntpath()` */ + strbuf_complete(resolved, '/'); + strbuf_addstr(resolved, append); + FREE_AND_NULL(append); + } + + return resolved->buf; + +} + char *mingw_getcwd(char *pointer, int len) { wchar_t cwd[MAX_PATH], wpointer[MAX_PATH]; @@ -1536,8 +1750,13 @@ char *mingw_getcwd(char *pointer, int len) if (hnd != INVALID_HANDLE_VALUE) { ret = GetFinalPathNameByHandleW(hnd, wpointer, ARRAY_SIZE(wpointer), 0); CloseHandle(hnd); - if (!ret || ret >= ARRAY_SIZE(wpointer)) - return NULL; + if (!ret || ret >= ARRAY_SIZE(wpointer)) { + ret = GetLongPathNameW(cwd, wpointer, ARRAY_SIZE(wpointer)); + if (!ret || ret >= ARRAY_SIZE(wpointer)) { + errno = ret ? ENAMETOOLONG : err_win_to_posix(GetLastError()); + return NULL; + } + } if (xwcstoutf(pointer, normalize_ntpath(wpointer), len) < 0) return NULL; return pointer; @@ -1648,7 +1867,7 @@ static const char *quote_arg_msys2(const char *arg) static const char *parse_interpreter(const char *cmd) { - static char buf[100]; + static char buf[MAX_PATH]; char *p, *opt; ssize_t n; /* read() can return negative values */ int fd; @@ -1708,6 +1927,65 @@ static char *lookup_prog(const char *dir, int dirlen, const char *cmd, return NULL; } +static char *path_lookup(const char *cmd, int exe_only); + +static char *is_busybox_applet(const char *cmd) +{ + static struct string_list applets = STRING_LIST_INIT_DUP; + static char *busybox_path; + static int busybox_path_initialized; + + /* Avoid infinite loop */ + if (!strncasecmp(cmd, "busybox", 7) && + (!cmd[7] || !strcasecmp(cmd + 7, ".exe"))) + return NULL; + + if (!busybox_path_initialized) { + busybox_path = path_lookup("busybox.exe", 1); + busybox_path_initialized = 1; + } + + /* Assume that sh is compiled in... */ + if (!busybox_path || !strcasecmp(cmd, "sh")) + return xstrdup_or_null(busybox_path); + + if (!applets.nr) { + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf buf = STRBUF_INIT; + char *p; + + strvec_pushl(&cp.args, busybox_path, "--help", NULL); + + if (capture_command(&cp, &buf, 2048)) { + string_list_append(&applets, ""); + return NULL; + } + + /* parse output */ + p = strstr(buf.buf, "Currently defined functions:\n"); + if (!p) { + warning("Could not parse output of busybox --help"); + string_list_append(&applets, ""); + return NULL; + } + p = strchrnul(p, '\n'); + for (;;) { + size_t len; + + p += strspn(p, "\n\t ,"); + len = strcspn(p, "\n\t ,"); + if (!len) + break; + p[len] = '\0'; + string_list_insert(&applets, p); + p = p + len + 1; + } + } + + return string_list_has_string(&applets, cmd) ? + xstrdup(busybox_path) : NULL; +} + /* * Determines the absolute path of cmd using the split path in path. * If cmd contains a slash or backslash, no lookup is performed. @@ -1736,6 +2014,9 @@ static char *path_lookup(const char *cmd, int exe_only) path = sep + 1; } + if (!prog && !isexe) + prog = is_busybox_applet(cmd); + return prog; } @@ -1939,8 +2220,8 @@ static int is_msys2_sh(const char *cmd) } static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaenv, - const char *dir, - int prepend_cmd, int fhin, int fhout, int fherr) + const char *dir, const char *prepend_cmd, + int fhin, int fhout, int fherr) { STARTUPINFOEXW si; PROCESS_INFORMATION pi; @@ -2008,6 +2289,10 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen if (*argv && !strcmp(cmd, *argv)) wcmd[0] = L'\0'; + /* + * Paths to executables and to the current directory do not support + * long paths, therefore we cannot use xutftowcs_long_path() here. + */ else if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -2016,9 +2301,9 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen /* concatenate argv, quoting args as we go */ strbuf_init(&args, 0); if (prepend_cmd) { - char *quoted = (char *)quote_arg(cmd); + char *quoted = (char *)quote_arg(prepend_cmd); strbuf_addstr(&args, quoted); - if (quoted != cmd) + if (quoted != prepend_cmd) free(quoted); } for (; *argv; argv++) { @@ -2138,7 +2423,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen return (pid_t)pi.dwProcessId; } -static pid_t mingw_spawnv(const char *cmd, const char **argv, int prepend_cmd) +static pid_t mingw_spawnv(const char *cmd, const char **argv, + const char *prepend_cmd) { return mingw_spawnve_fd(cmd, argv, NULL, NULL, prepend_cmd, 0, 1, 2); } @@ -2166,14 +2452,14 @@ pid_t mingw_spawnvpe(const char *cmd, const char **argv, char **deltaenv, pid = -1; } else { - pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, 1, + pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, interpr, fhin, fhout, fherr); free(iprog); } argv[0] = argv0; } else - pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, 0, + pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, NULL, fhin, fhout, fherr); free(prog); } @@ -2198,7 +2484,7 @@ static int try_shell_exec(const char *cmd, char *const *argv) argv2[0] = (char *)cmd; /* full path to the script file */ COPY_ARRAY(&argv2[1], &argv[1], argc); exec_id = trace2_exec(prog, (const char **)argv2); - pid = mingw_spawnv(prog, (const char **)argv2, 1); + pid = mingw_spawnv(prog, (const char **)argv2, interpr); if (pid >= 0) { int status; if (waitpid(pid, &status, 0) < 0) @@ -2222,7 +2508,7 @@ int mingw_execv(const char *cmd, char *const *argv) int exec_id; exec_id = trace2_exec(cmd, (const char **)argv); - pid = mingw_spawnv(cmd, (const char **)argv, 0); + pid = mingw_spawnv(cmd, (const char **)argv, NULL); if (pid < 0) { trace2_exec_result(exec_id, -1); return -1; @@ -2251,16 +2537,28 @@ int mingw_execvp(const char *cmd, char *const *argv) int mingw_kill(pid_t pid, int sig) { if (pid > 0 && sig == SIGTERM) { - HANDLE h = OpenProcess(PROCESS_TERMINATE, FALSE, pid); - - if (TerminateProcess(h, -1)) { + HANDLE h = OpenProcess(PROCESS_CREATE_THREAD | + PROCESS_QUERY_INFORMATION | + PROCESS_VM_OPERATION | PROCESS_VM_WRITE | + PROCESS_VM_READ | PROCESS_TERMINATE, + FALSE, pid); + int ret; + + if (h) + ret = exit_process(h, 128 + sig); + else { + h = OpenProcess(PROCESS_TERMINATE, FALSE, pid); + if (!h) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + ret = terminate_process_tree(h, 128 + sig); + } + if (ret) { + errno = err_win_to_posix(GetLastError()); CloseHandle(h); - return 0; } - - errno = err_win_to_posix(GetLastError()); - CloseHandle(h); - return -1; + return ret; } else if (pid > 0 && sig == 0) { HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid); if (h) { @@ -2384,18 +2682,235 @@ static void ensure_socket_initialization(void) initialized = 1; } +static int winsock_error_to_errno(DWORD err) +{ + switch (err) { + case WSAEINTR: return EINTR; + case WSAEBADF: return EBADF; + case WSAEACCES: return EACCES; + case WSAEFAULT: return EFAULT; + case WSAEINVAL: return EINVAL; + case WSAEMFILE: return EMFILE; + case WSAEWOULDBLOCK: return EWOULDBLOCK; + case WSAEINPROGRESS: return EINPROGRESS; + case WSAEALREADY: return EALREADY; + case WSAENOTSOCK: return ENOTSOCK; + case WSAEDESTADDRREQ: return EDESTADDRREQ; + case WSAEMSGSIZE: return EMSGSIZE; + case WSAEPROTOTYPE: return EPROTOTYPE; + case WSAENOPROTOOPT: return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: return EOPNOTSUPP; + case WSAEAFNOSUPPORT: return EAFNOSUPPORT; + case WSAEADDRINUSE: return EADDRINUSE; + case WSAEADDRNOTAVAIL: return EADDRNOTAVAIL; + case WSAENETDOWN: return ENETDOWN; + case WSAENETUNREACH: return ENETUNREACH; + case WSAENETRESET: return ENETRESET; + case WSAECONNABORTED: return ECONNABORTED; + case WSAECONNRESET: return ECONNRESET; + case WSAENOBUFS: return ENOBUFS; + case WSAEISCONN: return EISCONN; + case WSAENOTCONN: return ENOTCONN; + case WSAETIMEDOUT: return ETIMEDOUT; + case WSAECONNREFUSED: return ECONNREFUSED; + case WSAELOOP: return ELOOP; + case WSAENAMETOOLONG: return ENAMETOOLONG; + case WSAEHOSTUNREACH: return EHOSTUNREACH; + case WSAENOTEMPTY: return ENOTEMPTY; + /* No errno equivalent; default to EIO */ + case WSAESOCKTNOSUPPORT: + case WSAEPFNOSUPPORT: + case WSAESHUTDOWN: + case WSAETOOMANYREFS: + case WSAEHOSTDOWN: + case WSAEPROCLIM: + case WSAEUSERS: + case WSAEDQUOT: + case WSAESTALE: + case WSAEREMOTE: + case WSASYSNOTREADY: + case WSAVERNOTSUPPORTED: + case WSANOTINITIALISED: + case WSAEDISCON: + case WSAENOMORE: + case WSAECANCELLED: + case WSAEINVALIDPROCTABLE: + case WSAEINVALIDPROVIDER: + case WSAEPROVIDERFAILEDINIT: + case WSASYSCALLFAILURE: + case WSASERVICE_NOT_FOUND: + case WSATYPE_NOT_FOUND: + case WSA_E_NO_MORE: + case WSA_E_CANCELLED: + case WSAEREFUSED: + case WSAHOST_NOT_FOUND: + case WSATRY_AGAIN: + case WSANO_RECOVERY: + case WSANO_DATA: + case WSA_QOS_RECEIVERS: + case WSA_QOS_SENDERS: + case WSA_QOS_NO_SENDERS: + case WSA_QOS_NO_RECEIVERS: + case WSA_QOS_REQUEST_CONFIRMED: + case WSA_QOS_ADMISSION_FAILURE: + case WSA_QOS_POLICY_FAILURE: + case WSA_QOS_BAD_STYLE: + case WSA_QOS_BAD_OBJECT: + case WSA_QOS_TRAFFIC_CTRL_ERROR: + case WSA_QOS_GENERIC_ERROR: + case WSA_QOS_ESERVICETYPE: + case WSA_QOS_EFLOWSPEC: + case WSA_QOS_EPROVSPECBUF: + case WSA_QOS_EFILTERSTYLE: + case WSA_QOS_EFILTERTYPE: + case WSA_QOS_EFILTERCOUNT: + case WSA_QOS_EOBJLENGTH: + case WSA_QOS_EFLOWCOUNT: +#ifndef _MSC_VER + case WSA_QOS_EUNKNOWNPSOBJ: +#endif + case WSA_QOS_EPOLICYOBJ: + case WSA_QOS_EFLOWDESC: + case WSA_QOS_EPSFLOWSPEC: + case WSA_QOS_EPSFILTERSPEC: + case WSA_QOS_ESDMODEOBJ: + case WSA_QOS_ESHAPERATEOBJ: + case WSA_QOS_RESERVED_PETYPE: + default: return EIO; + } +} + +/* + * On Windows, `errno` is a global macro to a function call. + * This makes it difficult to debug and single-step our mappings. + */ +static inline void set_wsa_errno(void) +{ + DWORD wsa = WSAGetLastError(); + int e = winsock_error_to_errno(wsa); + errno = e; + +#ifdef DEBUG_WSA_ERRNO + fprintf(stderr, "winsock error: %d -> %d\n", wsa, e); + fflush(stderr); +#endif +} + +static inline int winsock_return(int ret) +{ + if (ret < 0) + set_wsa_errno(); + + return ret; +} + +#define WINSOCK_RETURN(x) do { return winsock_return(x); } while (0) + +#undef strerror +char *mingw_strerror(int errnum) +{ + static char buf[41] =""; + switch (errnum) { + case EWOULDBLOCK: + xsnprintf(buf, 41, "%s", "Operation would block"); + break; + case EINPROGRESS: + xsnprintf(buf, 41, "%s", "Operation now in progress"); + break; + case EALREADY: + xsnprintf(buf, 41, "%s", "Operation already in progress"); + break; + case ENOTSOCK: + xsnprintf(buf, 41, "%s", "Socket operation on non-socket"); + break; + case EDESTADDRREQ: + xsnprintf(buf, 41, "%s", "Destination address required"); + break; + case EMSGSIZE: + xsnprintf(buf, 41, "%s", "Message too long"); + break; + case EPROTOTYPE: + xsnprintf(buf, 41, "%s", "Protocol wrong type for socket"); + break; + case ENOPROTOOPT: + xsnprintf(buf, 41, "%s", "Protocol not available"); + break; + case EPROTONOSUPPORT: + xsnprintf(buf, 41, "%s", "Protocol not supported"); + break; + case EOPNOTSUPP: + xsnprintf(buf, 41, "%s", "Operation not supported"); + break; + case EAFNOSUPPORT: + xsnprintf(buf, 41, "%s", "Address family not supported by protocol"); + break; + case EADDRINUSE: + xsnprintf(buf, 41, "%s", "Address already in use"); + break; + case EADDRNOTAVAIL: + xsnprintf(buf, 41, "%s", "Cannot assign requested address"); + break; + case ENETDOWN: + xsnprintf(buf, 41, "%s", "Network is down"); + break; + case ENETUNREACH: + xsnprintf(buf, 41, "%s", "Network is unreachable"); + break; + case ENETRESET: + xsnprintf(buf, 41, "%s", "Network dropped connection on reset"); + break; + case ECONNABORTED: + xsnprintf(buf, 41, "%s", "Software caused connection abort"); + break; + case ECONNRESET: + xsnprintf(buf, 41, "%s", "Connection reset by peer"); + break; + case ENOBUFS: + xsnprintf(buf, 41, "%s", "No buffer space available"); + break; + case EISCONN: + xsnprintf(buf, 41, "%s", "Transport endpoint is already connected"); + break; + case ENOTCONN: + xsnprintf(buf, 41, "%s", "Transport endpoint is not connected"); + break; + case ETIMEDOUT: + xsnprintf(buf, 41, "%s", "Connection timed out"); + break; + case ECONNREFUSED: + xsnprintf(buf, 41, "%s", "Connection refused"); + break; + case ELOOP: + xsnprintf(buf, 41, "%s", "Too many levels of symbolic links"); + break; + case EHOSTUNREACH: + xsnprintf(buf, 41, "%s", "No route to host"); + break; + default: return strerror(errnum); + } + return buf; +} + #undef gethostname int mingw_gethostname(char *name, int namelen) { - ensure_socket_initialization(); - return gethostname(name, namelen); + ensure_socket_initialization(); + WINSOCK_RETURN(gethostname(name, namelen)); } #undef gethostbyname struct hostent *mingw_gethostbyname(const char *host) { + struct hostent *ret; + ensure_socket_initialization(); - return gethostbyname(host); + + ret = gethostbyname(host); + if (!ret) + set_wsa_errno(); + + return ret; } #undef getaddrinfo @@ -2403,7 +2918,7 @@ int mingw_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { ensure_socket_initialization(); - return getaddrinfo(node, service, hints, res); + WINSOCK_RETURN(getaddrinfo(node, service, hints, res)); } int mingw_socket(int domain, int type, int protocol) @@ -2414,16 +2929,7 @@ int mingw_socket(int domain, int type, int protocol) ensure_socket_initialization(); s = WSASocket(domain, type, protocol, NULL, 0, 0); if (s == INVALID_SOCKET) { - /* - * WSAGetLastError() values are regular BSD error codes - * biased by WSABASEERR. - * However, strerror() does not know about networking - * specific errors, which are values beginning at 38 or so. - * Therefore, we choose to leave the biased error code - * in errno so that _if_ someone looks up the code somewhere, - * then it is at least the number that are usually listed. - */ - errno = WSAGetLastError(); + set_wsa_errno(); return -1; } /* convert into a file descriptor */ @@ -2439,35 +2945,35 @@ int mingw_socket(int domain, int type, int protocol) int mingw_connect(int sockfd, struct sockaddr *sa, size_t sz) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return connect(s, sa, sz); + WINSOCK_RETURN(connect(s, sa, sz)); } #undef bind int mingw_bind(int sockfd, struct sockaddr *sa, size_t sz) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return bind(s, sa, sz); + WINSOCK_RETURN(bind(s, sa, sz)); } #undef setsockopt int mingw_setsockopt(int sockfd, int lvl, int optname, void *optval, int optlen) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return setsockopt(s, lvl, optname, (const char*)optval, optlen); + WINSOCK_RETURN(setsockopt(s, lvl, optname, (const char*)optval, optlen)); } #undef shutdown int mingw_shutdown(int sockfd, int how) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return shutdown(s, how); + WINSOCK_RETURN(shutdown(s, how)); } #undef listen int mingw_listen(int sockfd, int backlog) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return listen(s, backlog); + WINSOCK_RETURN(listen(s, backlog)); } #undef accept @@ -2478,6 +2984,11 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) SOCKET s1 = (SOCKET)_get_osfhandle(sockfd1); SOCKET s2 = accept(s1, sa, sz); + if (s2 == INVALID_SOCKET) { + set_wsa_errno(); + return -1; + } + /* convert into a file descriptor */ if ((sockfd2 = _open_osfhandle(s2, O_RDWR|O_BINARY)) < 0) { int err = errno; @@ -2492,14 +3003,14 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) int mingw_rename(const char *pold, const char *pnew) { static int supports_file_rename_info_ex = 1; - DWORD attrs = INVALID_FILE_ATTRIBUTES, gle; + DWORD attrs = INVALID_FILE_ATTRIBUTES, gle, attrsold; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; int wpnew_len; - if (xutftowcs_path(wpold, pold) < 0) + if (xutftowcs_long_path(wpold, pold) < 0) return -1; - wpnew_len = xutftowcs_path(wpnew, pnew); + wpnew_len = xutftowcs_long_path(wpnew, pnew); if (wpnew_len < 0) return -1; @@ -2529,9 +3040,9 @@ int mingw_rename(const char *pold, const char *pnew) * flex array so that the structure has to be allocated on * the heap. As we declare this structure ourselves though * we can avoid the allocation and define FileName to have - * MAX_PATH bytes. + * MAX_LONG_PATH bytes. */ - WCHAR FileName[MAX_PATH]; + WCHAR FileName[MAX_LONG_PATH]; } rename_info = { 0 }; HANDLE old_handle = INVALID_HANDLE_VALUE; BOOL success; @@ -2584,6 +3095,26 @@ int mingw_rename(const char *pold, const char *pnew) gle = GetLastError(); } + if (gle == ERROR_ACCESS_DENIED) { + if (is_inside_windows_container()) { + /* Fall back to copy to destination & remove source */ + if (CopyFileW(wpold, wpnew, FALSE) && !mingw_unlink(pold, 1)) + return 0; + gle = GetLastError(); + } else if ((attrsold = GetFileAttributesW(wpold)) & FILE_ATTRIBUTE_READONLY) { + /* if file is read-only, change and retry */ + SetFileAttributesW(wpold, attrsold & ~FILE_ATTRIBUTE_READONLY); + if (MoveFileExW(wpold, wpnew, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) { + SetFileAttributesW(wpnew, attrsold); + return 0; + } + gle = GetLastError(); + /* revert attribute change on failure */ + SetFileAttributesW(wpold, attrsold); + } + } + /* revert file attributes on failure */ if (attrs != INVALID_FILE_ATTRIBUTES) SetFileAttributesW(wpnew, attrs); @@ -2885,9 +3416,9 @@ int mingw_raise(int sig) int link(const char *oldpath, const char *newpath) { - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!CreateHardLinkW(wnewpath, woldpath, NULL)) { @@ -2897,9 +3428,40 @@ int link(const char *oldpath, const char *newpath) return 0; } -int symlink(const char *target, const char *link) +enum symlink_type { + SYMLINK_TYPE_UNSPECIFIED = 0, + SYMLINK_TYPE_FILE, + SYMLINK_TYPE_DIRECTORY, +}; + +static enum symlink_type check_symlink_attr(struct index_state *index, const char *link) +{ + static struct attr_check *check; + const char *value; + + if (!index) + return SYMLINK_TYPE_UNSPECIFIED; + + if (!check) + check = attr_check_initl("symlink", NULL); + + git_check_attr(index, link, check); + + value = check->items[0].value; + if (ATTR_UNSET(value)) + return SYMLINK_TYPE_UNSPECIFIED; + if (!strcmp(value, "file")) + return SYMLINK_TYPE_FILE; + if (!strcmp(value, "dir") || !strcmp(value, "directory")) + return SYMLINK_TYPE_DIRECTORY; + + warning(_("ignoring invalid symlink type '%s' for '%s'"), value, link); + return SYMLINK_TYPE_UNSPECIFIED; +} + +int mingw_create_symlink(struct index_state *index, const char *target, const char *link) { - wchar_t wtarget[MAX_PATH], wlink[MAX_PATH]; + wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; int len; /* fail if symlinks are disabled or API is not supported (WinXP) */ @@ -2908,8 +3470,8 @@ int symlink(const char *target, const char *link) return -1; } - if ((len = xutftowcs_path(wtarget, target)) < 0 - || xutftowcs_path(wlink, link) < 0) + if ((len = xutftowcs_long_path(wtarget, target)) < 0 + || xutftowcs_long_path(wlink, link) < 0) return -1; /* convert target dir separators to backslashes */ @@ -2917,58 +3479,41 @@ int symlink(const char *target, const char *link) if (wtarget[len] == '/') wtarget[len] = '\\'; - /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* convert to directory symlink if target exists */ - switch (process_phantom_symlink(wtarget, wlink)) { - case PHANTOM_SYMLINK_RETRY: { - /* if target doesn't exist, add to phantom symlinks list */ - wchar_t wfullpath[MAX_PATH]; - struct phantom_symlink_info *psi; - - /* convert to absolute path to be independent of cwd */ - len = GetFullPathNameW(wlink, MAX_PATH, wfullpath, NULL); - if (!len || len >= MAX_PATH) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* over-allocate and fill phantom_symlink_info structure */ - psi = xmalloc(sizeof(struct phantom_symlink_info) - + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); - psi->wlink = (wchar_t *)(psi + 1); - wcscpy(psi->wlink, wfullpath); - psi->wtarget = psi->wlink + len + 1; - wcscpy(psi->wtarget, wtarget); - - EnterCriticalSection(&phantom_symlinks_cs); - psi->next = phantom_symlinks; - phantom_symlinks = psi; - LeaveCriticalSection(&phantom_symlinks_cs); - break; - } - case PHANTOM_SYMLINK_DIRECTORY: - /* if we created a dir symlink, process other phantom symlinks */ + switch (check_symlink_attr(index, link)) { + case SYMLINK_TYPE_UNSPECIFIED: + /* Create a phantom symlink: it is initially created as a file + * symlink, but may change to a directory symlink later if/when + * the target exists. */ + return create_phantom_symlink(wtarget, wlink); + case SYMLINK_TYPE_FILE: + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) + break; + return 0; + case SYMLINK_TYPE_DIRECTORY: + if (!CreateSymbolicLinkW(wlink, wtarget, + symlink_directory_flags)) + break; + /* There may be dangling phantom symlinks that point at this + * one, which should now morph into directory symlinks. */ process_phantom_symlinks(); - break; + return 0; default: - break; + BUG("unhandled symlink type"); } - return 0; + + /* CreateSymbolicLinkW failed. */ + errno = err_win_to_posix(GetLastError()); + return -1; } int readlink(const char *path, char *buf, size_t bufsiz) { - WCHAR wpath[MAX_PATH]; - char tmpbuf[MAX_PATH]; + WCHAR wpath[MAX_LONG_PATH]; + char tmpbuf[MAX_LONG_PATH]; int len; DWORD tag; - if (xutftowcs_path(wpath, path) < 0) + if (xutftowcs_long_path(wpath, path) < 0) return -1; if (read_reparse_point(wpath, TRUE, tmpbuf, &len, &tag) < 0) @@ -3037,6 +3582,30 @@ pid_t waitpid(pid_t pid, int *status, int options) return -1; } +int (*win32_is_mount_point)(struct strbuf *path) = mingw_is_mount_point; + +int mingw_is_mount_point(struct strbuf *path) +{ + WIN32_FIND_DATAW findbuf = { 0 }; + HANDLE handle; + wchar_t wfilename[MAX_LONG_PATH]; + int wlen = xutftowcs_long_path(wfilename, path->buf); + if (wlen < 0) + die(_("could not get long path for '%s'"), path->buf); + + /* remove trailing slash, if any */ + if (wlen > 0 && wfilename[wlen - 1] == L'/') + wfilename[--wlen] = L'\0'; + + handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + return 0; + FindClose(handle); + + return (findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && + (findbuf.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT); +} + int xutftowcsn(wchar_t *wcs, const char *utfs, size_t wcslen, int utflen) { int upos = 0, wpos = 0; @@ -3122,6 +3691,57 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen) return -1; } +#ifdef ENSURE_MSYSTEM_IS_SET +#if !defined(RUNTIME_PREFIX) || !defined(HAVE_WPGMPTR) || !defined(MINGW_PREFIX) +static size_t append_system_bin_dirs(char *path UNUSED, size_t size UNUSED) +{ + return 0; +} +#else +static size_t append_system_bin_dirs(char *path, size_t size) +{ + char prefix[32768]; + const char *slash; + size_t len = xwcstoutf(prefix, _wpgmptr, sizeof(prefix)), off = 0; + + if (len == 0 || len >= sizeof(prefix) || + !(slash = find_last_dir_sep(prefix))) + return 0; + /* strip trailing `git.exe` */ + len = slash - prefix; + + /* strip trailing `cmd` or `\bin` or `bin` or `libexec\git-core` */ + if (strip_suffix_mem(prefix, &len, "\\" MINGW_PREFIX "\\libexec\\git-core") || + strip_suffix_mem(prefix, &len, "\\" MINGW_PREFIX "\\bin")) + off += xsnprintf(path + off, size - off, + "%.*s\\" MINGW_PREFIX "\\bin;", (int)len, prefix); + else if (strip_suffix_mem(prefix, &len, "\\cmd") || + strip_suffix_mem(prefix, &len, "\\bin") || + strip_suffix_mem(prefix, &len, "\\libexec\\git-core")) + off += xsnprintf(path + off, size - off, + "%.*s\\" MINGW_PREFIX "\\bin;", (int)len, prefix); + else + return 0; + + off += xsnprintf(path + off, size - off, + "%.*s\\usr\\bin;", (int)len, prefix); + return off; +} +#endif +#endif + +static int is_system32_path(const char *path) +{ + WCHAR system32[MAX_LONG_PATH], wpath[MAX_LONG_PATH]; + + if (xutftowcs_long_path(wpath, path) < 0 || + !GetSystemDirectoryW(system32, ARRAY_SIZE(system32)) || + _wcsicmp(system32, wpath)) + return 0; + + return 1; +} + static void setup_windows_environment(void) { char *tmp = getenv("TMPDIR"); @@ -3146,9 +3766,20 @@ static void setup_windows_environment(void) convert_slashes(tmp); } - /* simulate TERM to enable auto-color (see color.c) */ - if (!getenv("TERM")) - setenv("TERM", "cygwin", 1); + + /* + * Make sure TERM is set up correctly to enable auto-color + * (see color.c .) Use "cygwin" for older OS releases which + * works correctly with MSYS2 utilities on older consoles. + */ + if (!getenv("TERM")) { + if ((GetVersion() >> 16) < 15063) + setenv("TERM", "cygwin", 0); + else { + setenv("TERM", "xterm-256color", 0); + setenv("COLORTERM", "truecolor", 0); + } + } /* calculate HOME if not set */ if (!getenv("HOME")) { @@ -3162,7 +3793,8 @@ static void setup_windows_environment(void) strbuf_addstr(&buf, tmp); if ((tmp = getenv("HOMEPATH"))) { strbuf_addstr(&buf, tmp); - if (is_directory(buf.buf)) + if (!is_system32_path(buf.buf) && + is_directory(buf.buf)) setenv("HOME", buf.buf, 1); else tmp = NULL; /* use $USERPROFILE */ @@ -3174,6 +3806,35 @@ static void setup_windows_environment(void) setenv("HOME", tmp, 1); } + if (!getenv("PLINK_PROTOCOL")) + setenv("PLINK_PROTOCOL", "ssh", 0); + +#ifdef ENSURE_MSYSTEM_IS_SET + if (!(tmp = getenv("MSYSTEM")) || !tmp[0]) { + const char *home = getenv("HOME"), *path = getenv("PATH"); + char buf[32768]; + size_t off = 0; + + setenv("MSYSTEM", ENSURE_MSYSTEM_IS_SET, 1); + + if (home) + off += xsnprintf(buf + off, sizeof(buf) - off, + "%s\\bin;", home); + off += append_system_bin_dirs(buf + off, sizeof(buf) - off); + if (path) + off += xsnprintf(buf + off, sizeof(buf) - off, + "%s", path); + else if (off > 0) + buf[off - 1] = '\0'; + else + buf[0] = '\0'; + setenv("PATH", buf, 1); + } +#endif + + if (!getenv("LC_ALL") && !getenv("LC_CTYPE") && !getenv("LANG")) + setenv("LC_CTYPE", "C.UTF-8", 1); + /* * Change 'core.symlinks' default to false, unless native symlinks are * enabled in MSys2 (via 'MSYS=winsymlinks:nativestrict'). Thus we can @@ -3297,9 +3958,7 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report) DACL_SECURITY_INFORMATION, &sid, NULL, NULL, NULL, &descriptor); - if (err != ERROR_SUCCESS) - error(_("failed to get owner for '%s' (%ld)"), path, err); - else if (sid && IsValidSid(sid)) { + if (err == ERROR_SUCCESS && sid && IsValidSid(sid)) { /* Now, verify that the SID matches the current user's */ static PSID current_user_sid; static HANDLE linked_token; @@ -3511,6 +4170,73 @@ int is_valid_win32_path(const char *path, int allow_literal_nul) } } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + /* Be careful not to add a drive prefix if there was none */ + if (is_wdir_sep(path[0]) && + !is_wdir_sep(buf[0]) && buf[1] == L':' && is_wdir_sep(buf[2])) + wcscpy(path, buf + 2); + else + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + #if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from @@ -3610,7 +4336,14 @@ static void adjust_symlink_flags(void) symlink_file_flags |= 2; symlink_directory_flags |= 2; } +} +static BOOL WINAPI handle_ctrl_c(DWORD ctrl_type) +{ + if (ctrl_type != CTRL_C_EVENT) + return FALSE; /* we did not handle this */ + mingw_raise(SIGINT); + return TRUE; /* we did handle this */ } #ifdef _MSC_VER @@ -3647,8 +4380,11 @@ int wmain(int argc, const wchar_t **wargv) #endif #endif + SetConsoleCtrlHandler(handle_ctrl_c, TRUE); + maybe_redirect_std_handles(); adjust_symlink_flags(); + fsync_object_files = 1; /* determine size of argv and environ conversion buffer */ maxlen = wcslen(wargv[0]); @@ -3680,6 +4416,9 @@ int wmain(int argc, const wchar_t **wargv) InitializeCriticalSection(&pinfo_cs); InitializeCriticalSection(&phantom_symlinks_cs); + /* initialize critical section for fscache */ + InitializeCriticalSection(&fscache_cs); + /* set up default file mode and file modes for stdin/out/err */ _fmode = _O_BINARY; _setmode(_fileno(stdin), _O_BINARY); @@ -3689,6 +4428,9 @@ int wmain(int argc, const wchar_t **wargv) /* initialize Unicode console */ winansi_init(); + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + /* invoke the real main() using our utf8 version of argv. */ exit_status = main(argc, argv); @@ -3733,3 +4475,62 @@ int mingw_have_unix_sockets(void) return ret; } #endif + +/* + * Based on https://stackoverflow.com/questions/43002803 + * + * [HKLM\SYSTEM\CurrentControlSet\Services\cexecsvc] + * "DisplayName"="@%systemroot%\\system32\\cexecsvc.exe,-100" + * "ErrorControl"=dword:00000001 + * "ImagePath"=hex(2):25,00,73,00,79,00,73,00,74,00,65,00,6d,00,72,00,6f,00, + * 6f,00,74,00,25,00,5c,00,73,00,79,00,73,00,74,00,65,00,6d,00,33,00,32,00, + * 5c,00,63,00,65,00,78,00,65,00,63,00,73,00,76,00,63,00,2e,00,65,00,78,00, + * 65,00,00,00 + * "Start"=dword:00000002 + * "Type"=dword:00000010 + * "Description"="@%systemroot%\\system32\\cexecsvc.exe,-101" + * "ObjectName"="LocalSystem" + * "ServiceSidType"=dword:00000001 + */ +int is_inside_windows_container(void) +{ + static int inside_container = -1; /* -1 uninitialized */ + const char *key = "SYSTEM\\CurrentControlSet\\Services\\cexecsvc"; + HKEY handle = NULL; + + if (inside_container != -1) + return inside_container; + + inside_container = ERROR_SUCCESS == + RegOpenKeyExA(HKEY_LOCAL_MACHINE, key, 0, KEY_READ, &handle); + RegCloseKey(handle); + + return inside_container; +} + +int file_attr_to_st_mode (DWORD attr, DWORD tag, const char *path) +{ + int fMode = S_IREAD; + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && + tag == IO_REPARSE_TAG_SYMLINK) { + int flag = S_IFLNK; + char buf[MAX_LONG_PATH]; + + /* + * Windows containers' mapped volumes are marked as reparse + * points and look like symbolic links, but they are not. + */ + if (path && is_inside_windows_container() && + readlink(path, buf, sizeof(buf)) > 27 && + starts_with(buf, "/ContainerMappedDirectories/")) + flag = S_IFDIR; + + fMode |= flag; + } else if (attr & FILE_ATTRIBUTE_DIRECTORY) + fMode |= S_IFDIR; + else + fMode |= S_IFREG; + if (!(attr & FILE_ATTRIBUTE_READONLY)) + fMode |= S_IWRITE; + return fMode; +} diff --git a/compat/mingw.h b/compat/mingw.h index 444daedfa52469..807ee7b7e2e573 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,5 +1,8 @@ #include "mingw-posix.h" +extern int core_fscache; +int are_long_paths_enabled(void); + struct config_context; int mingw_core_config(const char *var, const char *value, const struct config_context *ctx, void *cb); @@ -36,9 +39,17 @@ static inline void convert_slashes(char *path) if (*path == '\\') *path = '/'; } +struct strbuf; +int mingw_is_mount_point(struct strbuf *path); +extern int (*win32_is_mount_point)(struct strbuf *path); +#define is_mount_point win32_is_mount_point +#define CAN_UNLINK_MOUNT_POINTS 1 #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email +struct strbuf; +char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path); +#define platform_strbuf_realpath mingw_strbuf_realpath /** * Verifies that the specified path is owned by the user running the @@ -68,6 +79,42 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report); int is_valid_win32_path(const char *path, int allow_literal_nul); #define is_valid_path(path) is_valid_win32_path(path, 0) +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -126,18 +173,46 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) } /** - * Simplified file system specific variant of xutftowcsn, assumes output - * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). */ -static inline int xutftowcs_path(wchar_t *wcs, const char *utf) +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); + int result = xutftowcsn(wcs, utf, wcslen, utflen); if (result < 0 && errno == ERANGE) errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); return result; } +/** + * Simplified file system specific variant of xutftowcsn, assumes output + * buffer size is MAX_PATH wide chars and input string is \0-terminated, + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... + */ +static inline int xutftowcs_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + are_long_paths_enabled()); +} + /** * Converts UTF-16LE encoded string to UTF-8. * @@ -213,3 +288,8 @@ int mingw_have_unix_sockets(void); #undef have_unix_sockets #define have_unix_sockets mingw_have_unix_sockets #endif + +/* + * Check current process is inside Windows Container. + */ +int is_inside_windows_container(void); diff --git a/compat/posix.h b/compat/posix.h index faaae1b6555d1b..9040b27e1f3dd7 100644 --- a/compat/posix.h +++ b/compat/posix.h @@ -45,7 +45,7 @@ #define UNUSED #endif -#ifdef __MINGW64__ +#if defined(__MINGW32__) || defined(__MINGW64__) #define _POSIX_C_SOURCE 1 #elif defined(__sun__) /* diff --git a/compat/terminal.c b/compat/terminal.c index 584f27bf7e1078..882b027e41e52b 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -418,6 +418,55 @@ static int getchar_with_timeout(int timeout) return getchar(); } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + strvec_pushv(&child.args, read_input); + child.in = -1; + child.out = -1; + child.silent_exec_failure = 1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -430,6 +479,15 @@ char *git_terminal_prompt(const char *prompt, int echo) int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result) + return result; + +#endif + input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) return NULL; diff --git a/compat/vcbuild/README b/compat/vcbuild/README index 29ec1d0f104b80..9ac9760397f479 100644 --- a/compat/vcbuild/README +++ b/compat/vcbuild/README @@ -6,7 +6,11 @@ The Steps to Build Git with VS2015 or VS2017 from the command line. Prompt or from an SDK bash window: $ cd - $ ./compat/vcbuild/vcpkg_install.bat + $ ./compat/vcbuild/vcpkg_install.bat x64-windows + + or + + $ ./compat/vcbuild/vcpkg_install.bat arm64-windows The vcpkg tools and all of the third-party sources will be installed in this folder: @@ -37,27 +41,17 @@ The Steps to Build Git with VS2015 or VS2017 from the command line. ================================================================ -Alternatively, run `make vcxproj` and then load the generated `git.sln` in -Visual Studio. The initial build will install the vcpkg system and build the +Alternatively, just open Git's top-level directory in Visual Studio, via +`File>Open>Folder...`. This will use CMake internally to generate the +project definitions. It will also install the vcpkg system and build the dependencies automatically. This will take a while. -Instead of generating the `git.sln` file yourself (which requires a full Git -for Windows SDK), you may want to consider fetching the `vs/master` branch of -https://github.com/git-for-windows/git instead (which is updated automatically -via CI running `make vcxproj`). The `vs/master` branch does not require a Git -for Windows to build, but you can run the test scripts in a regular Git Bash. - -Note that `make vcxproj` will automatically add and commit the generated `.sln` -and `.vcxproj` files to the repo. This is necessary to allow building a -fully-testable Git in Visual Studio, where a regular Git Bash can be used to -run the test scripts (as opposed to a full Git for Windows SDK): a number of -build targets, such as Git commands implemented as Unix shell scripts (where -`@@SHELL_PATH@@` and other placeholders are interpolated) require a full-blown -Git for Windows SDK (which is about 10x the size of a regular Git for Windows -installation). - -If your plan is to open a Pull Request with Git for Windows, it is a good idea -to drop this commit before submitting. +You can also generate the Visual Studio solution manually by downloading +and running CMake explicitly rather than letting Visual Studio doing +that implicitly. + +Another, deprecated option is to run `make vcxproj`. This option is +superseded by the CMake-based build, and will be removed at some point. ================================================================ The Steps of Build Git with VS2008 diff --git a/compat/vcbuild/find_vs_env.bat b/compat/vcbuild/find_vs_env.bat index b35d264c0e6bed..379b16296e09c2 100644 --- a/compat/vcbuild/find_vs_env.bat +++ b/compat/vcbuild/find_vs_env.bat @@ -99,6 +99,7 @@ REM ================================================================ SET sdk_dir=%WindowsSdkDir% SET sdk_ver=%WindowsSDKVersion% + SET sdk_ver_bin_dir=%WindowsSdkVerBinPath%%tgt% SET si=%sdk_dir%Include\%sdk_ver% SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" SET sl=%sdk_dir%lib\%sdk_ver% @@ -130,6 +131,7 @@ REM ================================================================ SET sdk_dir=%WindowsSdkDir% SET sdk_ver=%WindowsSDKVersion% + SET sdk_ver_bin_dir=%WindowsSdkVerBinPath%bin\amd64 SET si=%sdk_dir%Include\%sdk_ver% SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" -I"%si%winrt" SET sl=%sdk_dir%lib\%sdk_ver% @@ -160,6 +162,11 @@ REM ================================================================ echo msvc_includes=%msvc_includes% echo msvc_libs=%msvc_libs% + echo sdk_ver_bin_dir=%sdk_ver_bin_dir% + SET X1=%sdk_ver_bin_dir:C:=/C% + SET X2=%X1:\=/% + echo sdk_ver_bin_dir_msys=%X2% + echo sdk_includes=%sdk_includes% echo sdk_libs=%sdk_libs% diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bd824154be381..677d44e46f98d6 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -15,6 +15,7 @@ my @lflags = (); my $is_linking = 0; my $is_debug = 0; +my $is_gui = 0; while (@ARGV) { my $arg = shift @ARGV; if ("$arg" eq "-DDEBUG") { @@ -56,7 +57,8 @@ # need to use that instead? foreach my $flag (@lflags) { if ($flag =~ /^-LIBPATH:(.*)/) { - foreach my $l ("libcurl_imp.lib", "libcurl.lib") { + my $libcurl = $is_debug ? "libcurl-d.lib" : "libcurl.lib"; + foreach my $l ("libcurl_imp.lib", $libcurl) { if (-f "$1/$l") { $lib = $l; last; @@ -66,7 +68,11 @@ } push(@args, $lib); } elsif ("$arg" eq "-lexpat") { + if ($is_debug) { + push(@args, "libexpatd.lib"); + } else { push(@args, "libexpat.lib"); + } } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; push(@lflags, $arg); @@ -118,11 +124,23 @@ push(@cflags, "-wd4996"); } elsif ("$arg" =~ /^-W[a-z]/) { # let's ignore those + } elsif ("$arg" eq "-fno-stack-protector") { + # eat this + } elsif ("$arg" eq "-mwindows") { + $is_gui = 1; } else { push(@args, $arg); } } if ($is_linking) { + if ($is_gui) { + push(@args, "-ENTRY:wWinMainCRTStartup"); + push(@args, "-SUBSYSTEM:WINDOWS"); + } else { + push(@args, "-ENTRY:wmainCRTStartup"); + push(@args, "-SUBSYSTEM:CONSOLE"); + } + push(@args, @lflags); unshift(@args, "link.exe"); } else { diff --git a/compat/vcbuild/scripts/rc.pl b/compat/vcbuild/scripts/rc.pl new file mode 100644 index 00000000000000..7bca4cd81c6c63 --- /dev/null +++ b/compat/vcbuild/scripts/rc.pl @@ -0,0 +1,46 @@ +#!/usr/bin/perl -w +###################################################################### +# Compile Resources on Windows +# +# This is a wrapper to facilitate the compilation of Git with MSVC +# using GNU Make as the build system. So, instead of manipulating the +# Makefile into something nasty, just to support non-space arguments +# etc, we use this wrapper to fix the command line options +# +###################################################################### +use strict; +my @args = (); +my @input = (); + +while (@ARGV) { + my $arg = shift @ARGV; + if ("$arg" =~ /^-[dD]/) { + # GIT_VERSION gets passed with too many + # layers of dquote escaping. + $arg =~ s/\\"/"/g; + + push(@args, $arg); + + } elsif ("$arg" eq "-i") { + my $arg = shift @ARGV; + # TODO complain if NULL or is dashed ?? + push(@input, $arg); + + } elsif ("$arg" eq "-o") { + my $arg = shift @ARGV; + # TODO complain if NULL or is dashed ?? + push(@args, "-fo$arg"); + + } else { + push(@args, $arg); + } +} + +push(@args, "-nologo"); +push(@args, "-v"); +push(@args, @input); + +unshift(@args, "rc.exe"); +printf("**** @args\n"); + +exit (system(@args) != 0); diff --git a/compat/vcbuild/vcpkg_copy_dlls.bat b/compat/vcbuild/vcpkg_copy_dlls.bat index 13661c14f8705c..8bea0cbf83b6cf 100644 --- a/compat/vcbuild/vcpkg_copy_dlls.bat +++ b/compat/vcbuild/vcpkg_copy_dlls.bat @@ -15,7 +15,12 @@ REM ================================================================ @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD cd %cwd% - SET arch=x64-windows + SET arch=%2 + IF NOT DEFINED arch ( + echo defaulting to 'x64-windows`. Invoke %0 with 'x86-windows', 'x64-windows', or 'arm64-windows' + set arch=x64-windows + ) + SET inst=%cwd%vcpkg\installed\%arch% IF [%1]==[release] ( diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index ebd0bad242a8ca..575c65c20ba307 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -31,11 +31,24 @@ REM ================================================================ SETLOCAL EnableDelayedExpansion + SET arch=%1 + IF NOT DEFINED arch ( + echo defaulting to 'x64-windows`. Invoke %0 with 'x86-windows', 'x64-windows', or 'arm64-windows' + set arch=x64-windows + ) + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD cd %cwd% dir vcpkg\vcpkg.exe >nul 2>nul && GOTO :install_libraries + git.exe version 2>nul + IF ERRORLEVEL 1 ( + echo "***" + echo "Git not found. Please adjust your CMD path or Git install option." + echo "***" + EXIT /B 1 ) + echo Fetching vcpkg in %cwd%vcpkg git.exe clone https://github.com/Microsoft/vcpkg vcpkg IF ERRORLEVEL 1 ( EXIT /B 1 ) @@ -48,9 +61,8 @@ REM ================================================================ echo Successfully installed %cwd%vcpkg\vcpkg.exe :install_libraries - SET arch=x64-windows - echo Installing third-party libraries... + echo Installing third-party libraries(%arch%)... FOR %%i IN (zlib expat libiconv openssl libssh2 curl) DO ( cd %cwd%vcpkg IF NOT EXIST "packages\%%i_%arch%" CALL :sub__install_one %%i @@ -73,8 +85,47 @@ REM ================================================================ :sub__install_one echo Installing package %1... - .\vcpkg.exe install %1:%arch% + call :%1_features + + REM vcpkg may not be reliable on slow, intermittent or proxy + REM connections, see e.g. + REM https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/4a8f7be5-5e15-4213-a7bb-ddf424a954e6/winhttpsendrequest-ends-with-12002-errorhttptimeout-after-21-seconds-no-matter-what-timeout?forum=windowssdk + REM which explains the hidden 21 second timeout + REM (last post by Dave : Microsoft - Windows Networking team) + + .\vcpkg.exe install %1%features%:%arch% IF ERRORLEVEL 1 ( EXIT /B 1 ) echo Finished %1 goto :EOF + +:: +:: features for each vcpkg to install +:: there should be an entry here for each package to install +:: 'set features=' means use the default otherwise +:: 'set features=[comma-delimited-feature-set]' is the syntax +:: + +:zlib_features +set features= +goto :EOF + +:expat_features +set features= +goto :EOF + +:libiconv_features +set features= +goto :EOF + +:openssl_features +set features= +goto :EOF + +:libssh2_features +set features= +goto :EOF + +:curl_features +set features=[core,openssl,schannel] +goto :EOF diff --git a/compat/win32.h b/compat/win32.h index 671bcc81f93351..299f01bdf0f5a4 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,19 +6,7 @@ #include #endif -static inline int file_attr_to_st_mode (DWORD attr, DWORD tag) -{ - int fMode = S_IREAD; - if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) - fMode |= S_IFLNK; - else if (attr & FILE_ATTRIBUTE_DIRECTORY) - fMode |= S_IFDIR; - else - fMode |= S_IFREG; - if (!(attr & FILE_ATTRIBUTE_READONLY)) - fMode |= S_IWRITE; - return fMode; -} +extern int file_attr_to_st_mode (DWORD attr, DWORD tag, const char *path); static inline int get_file_attr(const char *fname, WIN32_FILE_ATTRIBUTE_DATA *fdata) { diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 24ee9b814d6adf..87063101f57202 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,21 @@ #include "../../git-compat-util.h" -struct DIR { - struct dirent dd_dir; /* includes d_type */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + struct dirent dd_dir; /* includes d_type */ +} dirent_DIR; +#pragma GCC diagnostic pop + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) @@ -21,41 +27,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -82,7 +54,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -93,3 +65,44 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *dirent_opendir(const char *name) +{ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + dirent_DIR *dir; + + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, + are_long_paths_enabled())) < 0) + return NULL; + + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '\\'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(dirent_DIR) + MAX_LONG_PATH); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return (DIR*) dir; +} diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bfed62..a58a8075fd70e3 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,34 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char d_name[/* FLEX_ARRAY */]; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +#define opendir git_opendir + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ diff --git a/compat/win32/exit-process.h b/compat/win32/exit-process.h new file mode 100644 index 00000000000000..d53989884cfb0c --- /dev/null +++ b/compat/win32/exit-process.h @@ -0,0 +1,165 @@ +#ifndef EXIT_PROCESS_H +#define EXIT_PROCESS_H + +/* + * This file contains functions to terminate a Win32 process, as gently as + * possible. + * + * At first, we will attempt to inject a thread that calls ExitProcess(). If + * that fails, we will fall back to terminating the entire process tree. + * + * For simplicity, these functions are marked as file-local. + */ + +#include + +/* + * Terminates the process corresponding to the process ID and all of its + * directly and indirectly spawned subprocesses. + * + * This way of terminating the processes is not gentle: the processes get + * no chance of cleaning up after themselves (closing file handles, removing + * .lock files, terminating spawned processes (if any), etc). + */ +static int terminate_process_tree(HANDLE main_process, int exit_status) +{ + HANDLE snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + PROCESSENTRY32 entry; + DWORD pids[16384]; + int max_len = sizeof(pids) / sizeof(*pids), i, len, ret = 0; + pid_t pid = GetProcessId(main_process); + + pids[0] = (DWORD)pid; + len = 1; + + /* + * Even if Process32First()/Process32Next() seem to traverse the + * processes in topological order (i.e. parent processes before + * child processes), there is nothing in the Win32 API documentation + * suggesting that this is guaranteed. + * + * Therefore, run through them at least twice and stop when no more + * process IDs were added to the list. + */ + for (;;) { + int orig_len = len; + + memset(&entry, 0, sizeof(entry)); + entry.dwSize = sizeof(entry); + + if (!Process32First(snapshot, &entry)) + break; + + do { + for (i = len - 1; i >= 0; i--) { + if (pids[i] == entry.th32ProcessID) + break; + if (pids[i] == entry.th32ParentProcessID) + pids[len++] = entry.th32ProcessID; + } + } while (len < max_len && Process32Next(snapshot, &entry)); + + if (orig_len == len || len >= max_len) + break; + } + + for (i = len - 1; i > 0; i--) { + HANDLE process = OpenProcess(PROCESS_TERMINATE, FALSE, pids[i]); + + if (process) { + if (!TerminateProcess(process, exit_status)) + ret = -1; + CloseHandle(process); + } + } + if (!TerminateProcess(main_process, exit_status)) + ret = -1; + CloseHandle(main_process); + + return ret; +} + +/** + * Determine whether a process runs in the same architecture as the current + * one. That test is required before we assume that GetProcAddress() returns + * a valid address *for the target process*. + */ +static inline int process_architecture_matches_current(HANDLE process) +{ + static BOOL current_is_wow = -1; + BOOL is_wow; + + if (current_is_wow == -1 && + !IsWow64Process (GetCurrentProcess(), ¤t_is_wow)) + current_is_wow = -2; + if (current_is_wow == -2) + return 0; /* could not determine current process' WoW-ness */ + if (!IsWow64Process (process, &is_wow)) + return 0; /* cannot determine */ + return is_wow == current_is_wow; +} + +/** + * Inject a thread into the given process that runs ExitProcess(). + * + * Note: as kernel32.dll is loaded before any process, the other process and + * this process will have ExitProcess() at the same address. + * + * This function expects the process handle to have the access rights for + * CreateRemoteThread(): PROCESS_CREATE_THREAD, PROCESS_QUERY_INFORMATION, + * PROCESS_VM_OPERATION, PROCESS_VM_WRITE, and PROCESS_VM_READ. + * + * The idea comes from the Dr Dobb's article "A Safer Alternative to + * TerminateProcess()" by Andrew Tucker (July 1, 1999), + * http://www.drdobbs.com/a-safer-alternative-to-terminateprocess/184416547 + * + * If this method fails, we fall back to running terminate_process_tree(). + */ +static int exit_process(HANDLE process, int exit_code) +{ + DWORD code; + + if (GetExitCodeProcess(process, &code) && code == STILL_ACTIVE) { + static int initialized; + static LPTHREAD_START_ROUTINE exit_process_address; + PVOID arg = (PVOID)(intptr_t)exit_code; + DWORD thread_id; + HANDLE thread = NULL; + + if (!initialized) { + HINSTANCE kernel32 = GetModuleHandleA("kernel32"); + if (!kernel32) + die("BUG: cannot find kernel32"); + exit_process_address = + (LPTHREAD_START_ROUTINE)(void (*)(void)) + GetProcAddress(kernel32, "ExitProcess"); + initialized = 1; + } + if (!exit_process_address || + !process_architecture_matches_current(process)) + return terminate_process_tree(process, exit_code); + + thread = CreateRemoteThread(process, NULL, 0, + exit_process_address, + arg, 0, &thread_id); + if (thread) { + CloseHandle(thread); + /* + * If the process survives for 10 seconds (a completely + * arbitrary value picked from thin air), fall back to + * killing the process tree via TerminateProcess(). + */ + if (WaitForSingleObject(process, 10000) == + WAIT_OBJECT_0) { + CloseHandle(process); + return 0; + } + } + + return terminate_process_tree(process, exit_code); + } + + return 0; +} + +#endif diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 00000000000000..cbd90ececf6b37 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,820 @@ +#include "../../git-compat-util.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" +#include "../../dir.h" +#include "../../abspath.h" +#include "../../trace.h" +#include "config.h" +#include "../../mem-pool.h" +#include "ntifs.h" +#include "wsl.h" + +static volatile long initialized; +static DWORD dwTlsIndex; +CRITICAL_SECTION fscache_cs; + +/* + * Store one fscache per thread to avoid thread contention and locking. + * This is ok because multi-threaded access is 1) uncommon and 2) always + * splitting up the cache entries across multiple threads so there isn't + * any overlap between threads anyway. + */ +struct fscache { + volatile long enabled; + struct hashmap map; + struct mem_pool mem_pool; + unsigned int lstat_requests; + unsigned int opendir_requests; + unsigned int fscache_requests; + unsigned int fscache_misses; + /* + * 32k wide characters translates to 64kB, which is the maximum that + * Windows 8.1 and earlier can handle. On network drives, not only + * the client's Windows version matters, but also the server's, + * therefore we need to keep this to 64kB. + */ + WCHAR buffer[32 * 1024]; +}; +static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + ULONG reparse_tag; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; + } s; + } u; + + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + struct dirent dirent; +}; +#pragma GCC diagnostic pop + +#pragma GCC diagnostic push +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif +struct heap_fsentry { + union { + struct fsentry ent; + char dummy[sizeof(struct fsentry) + MAX_LONG_PATH]; + } u; +}; +#pragma GCC diagnostic pop + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(void *cmp_data UNUSED, + const struct fsentry *fse1, const struct fsentry *fse2, + void *keydata UNUSED) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && + (res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2, NULL))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return fspathncmp(fse1->dirent.d_name, fse2->dirent.d_name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->dirent.d_name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + if (len > MAX_LONG_PATH) + BUG("Trying to allocate fsentry for long path '%.*s'", + (int)len, name); + memcpy(fse->dirent.d_name, name, len); + fse->dirent.d_name[len] = 0; + fse->len = len; + hashmap_entry_init(&fse->ent, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fscache *cache, struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = + mem_pool_alloc(&cache->mem_pool, sizeof(*fse) + len + 1); + /* init the rest of the structure */ + fsentry_init(fse, list, name, len); + fse->next = NULL; + fse->u.refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->u.refcnt)); +} + +/* + * Release the reference to an fsentry. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedDecrement(&(fse->u.refcnt)); +} + +static int xwcstoutfn(char *utf, int utflen, const wchar_t *wcs, int wcslen) +{ + if (!wcs || !utf || utflen < 1) { + errno = EINVAL; + return -1; + } + utflen = WideCharToMultiByte(CP_UTF8, 0, wcs, wcslen, utf, utflen, NULL, NULL); + if (utflen) + return utflen; + errno = ERANGE; + return -1; +} + +/* + * Allocate and initialize an fsentry from a FILE_FULL_DIR_INFORMATION structure. + */ +static struct fsentry *fseentry_create_entry(struct fscache *cache, + struct fsentry *list, + PFILE_FULL_DIR_INFORMATION fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + + len = xwcstoutfn(buf, ARRAY_SIZE(buf), fdata->FileName, fdata->FileNameLength / sizeof(wchar_t)); + + fse = fsentry_alloc(cache, list, buf, len); + + fse->reparse_tag = + fdata->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT ? + fdata->EaSize : 0; + + /* + * On certain Windows versions, host directories mapped into + * Windows Containers ("Volumes", see https://docs.docker.com/storage/volumes/) + * look like symbolic links, but their targets are paths that + * are valid only in kernel mode. + * + * Let's work around this by detecting that situation and + * telling Git that these are *not* symbolic links. + */ + if (fse->reparse_tag == IO_REPARSE_TAG_SYMLINK && + sizeof(buf) > (size_t)(list ? list->len + 1 : 0) + fse->len + 1 && + is_inside_windows_container()) { + size_t off = 0; + if (list) { + memcpy(buf, list->dirent.d_name, list->len); + buf[list->len] = '/'; + off = list->len + 1; + } + memcpy(buf + off, fse->dirent.d_name, fse->len); + buf[off + fse->len] = '\0'; + } + + fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes, + fdata->EaSize, buf); + fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : + S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK; + fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : + fdata->EndOfFile.LowPart | + (((off_t)fdata->EndOfFile.HighPart) << 32); + filetime_to_timespec((FILETIME *)&(fdata->LastAccessTime), + &(fse->u.s.st_atim)); + filetime_to_timespec((FILETIME *)&(fdata->LastWriteTime), + &(fse->u.s.st_mtim)); + filetime_to_timespec((FILETIME *)&(fdata->CreationTime), + &(fse->u.s.st_ctim)); + if (fdata->EaSize > 0 && + sizeof(buf) >= (size_t)(list ? list->len+1 : 0) + fse->len+1 && + are_wsl_compatible_mode_bits_enabled()) { + size_t off = 0; + wchar_t wpath[MAX_LONG_PATH]; + if (list && list->len) { + memcpy(buf, list->dirent.d_name, list->len); + buf[list->len] = '/'; + off = list->len + 1; + } + memcpy(buf + off, fse->dirent.d_name, fse->len); + buf[off + fse->len] = '\0'; + if (xutftowcs_long_path(wpath, buf) >= 0) + copy_wsl_mode_bits_from_disk(wpath, -1, &fse->st_mode); + } + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir, + int *dir_not_found) +{ + wchar_t pattern[MAX_LONG_PATH]; + NTSTATUS status; + IO_STATUS_BLOCK iosb; + PFILE_FULL_DIR_INFORMATION di; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + *dir_not_found = 0; + + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->dirent.d_name, + MAX_LONG_PATH, dir->len, MAX_PATH - 2, + are_long_paths_enabled())) < 0) + return NULL; + + /* handle CWD */ + if (!wlen) { + wlen = GetCurrentDirectoryW(ARRAY_SIZE(pattern), pattern); + if (!wlen || wlen >= (ssize_t)ARRAY_SIZE(pattern)) { + errno = wlen ? ENAMETOOLONG : err_win_to_posix(GetLastError()); + return NULL; + } + } + + h = CreateFileW(pattern, FILE_LIST_DIRECTORY, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + *dir_not_found = 1; /* or empty directory */ + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + trace_printf_key(&trace_fscache, "fscache: error(%d) '%s'\n", + errno, dir->dirent.d_name); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(cache, NULL, dir->dirent.d_name, dir->len); + list->st_mode = S_IFDIR; + list->dirent.d_type = DT_DIR; + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + status = NtQueryDirectoryFile(h, NULL, 0, 0, &iosb, cache->buffer, + sizeof(cache->buffer), FileFullDirectoryInformation, FALSE, NULL, FALSE); + if (!NT_SUCCESS(status)) { + /* + * NtQueryDirectoryFile returns STATUS_INVALID_PARAMETER when + * asked to enumerate an invalid directory (ie it is a file + * instead of a directory). Verify that is the actual cause + * of the error. + */ + if (status == (NTSTATUS)STATUS_INVALID_PARAMETER) { + DWORD attributes = GetFileAttributesW(pattern); + if (!(attributes & FILE_ATTRIBUTE_DIRECTORY)) + status = ERROR_DIRECTORY; + } + goto Error; + } + di = (PFILE_FULL_DIR_INFORMATION)(cache->buffer); + for (;;) { + + *phead = fseentry_create_entry(cache, list, di); + phead = &(*phead)->next; + + /* If there is no offset in the entry, the buffer has been exhausted. */ + if (di->NextEntryOffset == 0) { + status = NtQueryDirectoryFile(h, NULL, 0, 0, &iosb, cache->buffer, + sizeof(cache->buffer), FileFullDirectoryInformation, FALSE, NULL, FALSE); + if (!NT_SUCCESS(status)) { + if (status == STATUS_NO_MORE_FILES) + break; + goto Error; + } + + di = (PFILE_FULL_DIR_INFORMATION)(cache->buffer); + continue; + } + + /* Advance to the next entry. */ + di = (PFILE_FULL_DIR_INFORMATION)(((PUCHAR)di) + di->NextEntryOffset); + } + + CloseHandle(h); + return list; + +Error: + trace_printf_key(&trace_fscache, + "fscache: status(%ld) unable to query directory " + "contents '%s'\n", status, dir->dirent.d_name); + CloseHandle(h); + fsentry_release(list); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fscache *cache, struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&cache->map, &fse->ent); +} + +/* + * Clears the cache. + */ +static void fscache_clear(struct fscache *cache) +{ + mem_pool_discard(&cache->mem_pool, 0); + mem_pool_init(&cache->mem_pool, 0); + hashmap_clear(&cache->map); + hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); + cache->lstat_requests = cache->opendir_requests = 0; + cache->fscache_misses = cache->fscache_requests = 0; +} + +/* + * Checks if the cache is enabled for the given path. + */ +static int do_fscache_enabled(struct fscache *cache, const char *path) +{ + return cache->enabled > 0 && !is_absolute_path(path); +} + +int fscache_enabled(const char *path) +{ + struct fscache *cache = fscache_getcache(); + + return cache ? do_fscache_enabled(cache, path) : 0; +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) +{ + struct fsentry *fse; + int dir_not_found; + + cache->fscache_requests++; + /* check if entry is in cache */ + fse = hashmap_get_entry(&cache->map, key, ent, NULL); + if (fse) { + if (fse->st_mode) + fsentry_addref(fse); + else + fse = NULL; /* non-existing directory */ + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get_entry(&cache->map, key->list, ent, NULL); + if (fse) { + /* + * dir entry without file entry, or dir does not + * exist -> file doesn't exist + */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing */ + fse = fsentry_create_list(cache, key->list ? key->list : key, &dir_not_found); + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + if (dir_not_found && key->list) { + /* + * Record that the directory does not exist (or is + * empty, which for all practical matters is the same + * thing as far as fscache is concerned). + */ + fse = fsentry_alloc(cache, key->list->list, + key->list->dirent.d_name, + key->list->len); + fse->st_mode = 0; + hashmap_add(&cache->map, &fse->ent); + } + return NULL; + } + + /* add directory listing to the cache */ + cache->fscache_misses++; + fscache_add(cache, fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get_entry(&cache->map, key, ent, NULL); + + if (fse && !fse->st_mode) + fse = NULL; /* non-existing directory */ + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + return fse; +} + +/* + * Enables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(size_t initial_size) +{ + int fscache; + struct fscache *cache; + int result = 0; + + /* allow the cache to be disabled entirely */ + fscache = git_env_bool("GIT_TEST_FSCACHE", -1); + if (fscache != -1) + core_fscache = fscache; + if (!core_fscache) + return 0; + + /* + * refcount the global fscache initialization so that the + * opendir and lstat function pointers are redirected if + * any threads are using the fscache. + */ + EnterCriticalSection(&fscache_cs); + if (!initialized) { + if (!dwTlsIndex) { + dwTlsIndex = TlsAlloc(); + if (dwTlsIndex == TLS_OUT_OF_INDEXES) { + LeaveCriticalSection(&fscache_cs); + return 0; + } + } + + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + win32_is_mount_point = fscache_is_mount_point; + } + initialized++; + LeaveCriticalSection(&fscache_cs); + + /* refcount the thread specific initialization */ + cache = fscache_getcache(); + if (cache) { + cache->enabled++; + } else { + cache = (struct fscache *)xcalloc(1, sizeof(*cache)); + cache->enabled = 1; + /* + * avoid having to rehash by leaving room for the parent dirs. + * '4' was determined empirically by testing several repos + */ + hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, initial_size * 4); + mem_pool_init(&cache->mem_pool, 0); + if (!TlsSetValue(dwTlsIndex, cache)) + BUG("TlsSetValue error"); + } + + trace_printf_key(&trace_fscache, "fscache: enable\n"); + return result; +} + +/* + * Disables the cache. + */ +void fscache_disable(void) +{ + struct fscache *cache; + + if (!core_fscache) + return; + + /* update the thread specific fscache initialization */ + cache = fscache_getcache(); + if (!cache) + BUG("fscache_disable() called on a thread where fscache has not been initialized"); + if (!cache->enabled) + BUG("fscache_disable() called on an fscache that is already disabled"); + cache->enabled--; + if (!cache->enabled) { + TlsSetValue(dwTlsIndex, NULL); + trace_printf_key(&trace_fscache, "fscache_disable: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + cache->lstat_requests, cache->opendir_requests, + cache->fscache_requests, cache->fscache_misses); + mem_pool_discard(&cache->mem_pool, 0); + hashmap_clear(&cache->map); + free(cache); + } + + /* update the global fscache initialization */ + EnterCriticalSection(&fscache_cs); + initialized--; + if (!initialized) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + win32_is_mount_point = mingw_is_mount_point; + } + LeaveCriticalSection(&fscache_cs); + + trace_printf_key(&trace_fscache, "fscache: disable\n"); + return; +} + +/* + * Flush cached stats result when fscache is enabled. + */ +void fscache_flush(void) +{ + struct fscache *cache = fscache_getcache(); + + if (cache && cache->enabled) { + fscache_clear(cache); + } +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; +#pragma GCC diagnostic push +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif + struct heap_fsentry key[2]; +#pragma GCC diagnostic pop + struct fsentry *fse; + struct fscache *cache = fscache_getcache(); + + if (!cache || !do_fscache_enabled(cache, filename)) + return mingw_lstat(filename, st); + + cache->lstat_requests++; + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(&key[0].u.ent, NULL, filename, dirlen); + fsentry_init(&key[1].u.ent, &key[0].u.ent, filename + base, len - base); + fse = fscache_get(cache, &key[1].u.ent); + if (!fse) { + errno = ENOENT; + return -1; + } + + /* + * Special case symbolic links: FindFirstFile()/FindNextFile() did not + * provide us with the length of the target path. + */ + if (fse->u.s.st_size == MAX_LONG_PATH && S_ISLNK(fse->st_mode)) { + char buf[MAX_LONG_PATH]; + int len = readlink(filename, buf, sizeof(buf) - 1); + + if (len > 0) + fse->u.s.st_size = len; + } + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->u.s.st_size; + st->st_atim = fse->u.s.st_atim; + st->st_mtim = fse->u.s.st_mtim; + st->st_ctim = fse->u.s.st_ctim; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +/* + * is_mount_point() replacement, uses cache if enabled, otherwise falls + * back to mingw_is_mount_point(). + */ +int fscache_is_mount_point(struct strbuf *path) +{ + int dirlen, base, len; +#pragma GCC diagnostic push +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif + struct heap_fsentry key[2]; +#pragma GCC diagnostic pop + struct fsentry *fse; + struct fscache *cache = fscache_getcache(); + + if (!cache || !do_fscache_enabled(cache, path->buf)) + return mingw_is_mount_point(path); + + cache->lstat_requests++; + /* split path into path + name */ + len = path->len; + if (len && is_dir_sep(path->buf[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(path->buf[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(&key[0].u.ent, NULL, path->buf, dirlen); + fsentry_init(&key[1].u.ent, &key[0].u.ent, path->buf + base, len - base); + fse = fscache_get(cache, &key[1].u.ent); + if (!fse) + return mingw_is_mount_point(path); + return fse->reparse_tag == IO_REPARSE_TAG_MOUNT_POINT; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent *dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent = &next->dirent; + return dir->dirent; +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct heap_fsentry key; + struct fsentry *list; + fscache_DIR *dir; + int len; + struct fscache *cache = fscache_getcache(); + + if (!cache || !do_fscache_enabled(cache, dirname)) + return dirent_opendir(dirname); + + cache->opendir_requests++; + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key.u.ent, NULL, dirname, len); + list = fscache_get(cache, &key.u.ent); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} + +struct fscache *fscache_getcache(void) +{ + return (struct fscache *)TlsGetValue(dwTlsIndex); +} + +void fscache_merge(struct fscache *dest) +{ + struct hashmap_iter iter; + struct hashmap_entry *e; + struct fscache *cache = fscache_getcache(); + + /* + * Only do the merge if fscache was enabled and we have a dest + * cache to merge into. + */ + if (!dest) { + fscache_enable(0); + return; + } + if (!cache) + BUG("fscache_merge() called on a thread where fscache has not been initialized"); + + TlsSetValue(dwTlsIndex, NULL); + trace_printf_key(&trace_fscache, "fscache_merge: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + cache->lstat_requests, cache->opendir_requests, + cache->fscache_requests, cache->fscache_misses); + + /* + * This is only safe because the primary thread we're merging into + * isn't being used so the critical section only needs to prevent + * the the child threads from stomping on each other. + */ + EnterCriticalSection(&fscache_cs); + + hashmap_iter_init(&cache->map, &iter); + while ((e = hashmap_iter_next(&iter))) + hashmap_add(&dest->map, e); + + mem_pool_combine(&dest->mem_pool, &cache->mem_pool); + + dest->lstat_requests += cache->lstat_requests; + dest->opendir_requests += cache->opendir_requests; + dest->fscache_requests += cache->fscache_requests; + dest->fscache_misses += cache->fscache_misses; + initialized--; + LeaveCriticalSection(&fscache_cs); + + free(cache); + +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 00000000000000..386c770a85d321 --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,36 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +/* + * The fscache is thread specific. enable_fscache() must be called + * for each thread where caching is desired. + */ + +extern CRITICAL_SECTION fscache_cs; + +int fscache_enable(size_t initial_size); +#define enable_fscache(initial_size) fscache_enable(initial_size) + +void fscache_disable(void); +#define disable_fscache() fscache_disable() + +int fscache_enabled(const char *path); +#define is_fscache_enabled(path) fscache_enabled(path) + +void fscache_flush(void); +#define flush_fscache() fscache_flush() + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); +int fscache_is_mount_point(struct strbuf *path); + +/* opaque fscache structure */ +struct fscache; + +struct fscache *fscache_getcache(void); +#define getcache_fscache() fscache_getcache() + +void fscache_merge(struct fscache *dest); +#define merge_fscache(dest) fscache_merge(dest) + +#endif diff --git a/compat/win32/ntifs.h b/compat/win32/ntifs.h new file mode 100644 index 00000000000000..64ed792c52f352 --- /dev/null +++ b/compat/win32/ntifs.h @@ -0,0 +1,131 @@ +#ifndef _NTIFS_ +#define _NTIFS_ + +/* + * Copy necessary structures and definitions out of the Windows DDK + * to enable calling NtQueryDirectoryFile() + */ + +typedef _Return_type_success_(return >= 0) LONG NTSTATUS; +#define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) + +#if !defined(_NTSECAPI_) && !defined(_WINTERNL_) && \ + !defined(__UNICODE_STRING_DEFINED) +#define __UNICODE_STRING_DEFINED +typedef struct _UNICODE_STRING { + USHORT Length; + USHORT MaximumLength; + PWSTR Buffer; +} UNICODE_STRING; +typedef UNICODE_STRING *PUNICODE_STRING; +typedef const UNICODE_STRING *PCUNICODE_STRING; +#endif /* !_NTSECAPI_ && !_WINTERNL_ && !__UNICODE_STRING_DEFINED */ + +typedef enum _FILE_INFORMATION_CLASS { + FileDirectoryInformation = 1, + FileFullDirectoryInformation, + FileBothDirectoryInformation, + FileBasicInformation, + FileStandardInformation, + FileInternalInformation, + FileEaInformation, + FileAccessInformation, + FileNameInformation, + FileRenameInformation, + FileLinkInformation, + FileNamesInformation, + FileDispositionInformation, + FilePositionInformation, + FileFullEaInformation, + FileModeInformation, + FileAlignmentInformation, + FileAllInformation, + FileAllocationInformation, + FileEndOfFileInformation, + FileAlternateNameInformation, + FileStreamInformation, + FilePipeInformation, + FilePipeLocalInformation, + FilePipeRemoteInformation, + FileMailslotQueryInformation, + FileMailslotSetInformation, + FileCompressionInformation, + FileObjectIdInformation, + FileCompletionInformation, + FileMoveClusterInformation, + FileQuotaInformation, + FileReparsePointInformation, + FileNetworkOpenInformation, + FileAttributeTagInformation, + FileTrackingInformation, + FileIdBothDirectoryInformation, + FileIdFullDirectoryInformation, + FileValidDataLengthInformation, + FileShortNameInformation, + FileIoCompletionNotificationInformation, + FileIoStatusBlockRangeInformation, + FileIoPriorityHintInformation, + FileSfioReserveInformation, + FileSfioVolumeInformation, + FileHardLinkInformation, + FileProcessIdsUsingFileInformation, + FileNormalizedNameInformation, + FileNetworkPhysicalNameInformation, + FileIdGlobalTxDirectoryInformation, + FileIsRemoteDeviceInformation, + FileAttributeCacheInformation, + FileNumaNodeInformation, + FileStandardLinkInformation, + FileRemoteProtocolInformation, + FileMaximumInformation +} FILE_INFORMATION_CLASS, *PFILE_INFORMATION_CLASS; + +typedef struct _FILE_FULL_DIR_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + ULONG EaSize; + WCHAR FileName[1]; +} FILE_FULL_DIR_INFORMATION, *PFILE_FULL_DIR_INFORMATION; + +typedef struct _IO_STATUS_BLOCK { + union { + NTSTATUS Status; + PVOID Pointer; + } u; + ULONG_PTR Information; +} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK; + +typedef VOID +(NTAPI *PIO_APC_ROUTINE)( + IN PVOID ApcContext, + IN PIO_STATUS_BLOCK IoStatusBlock, + IN ULONG Reserved); + +NTSYSCALLAPI +NTSTATUS +NTAPI +NtQueryDirectoryFile( + _In_ HANDLE FileHandle, + _In_opt_ HANDLE Event, + _In_opt_ PIO_APC_ROUTINE ApcRoutine, + _In_opt_ PVOID ApcContext, + _Out_ PIO_STATUS_BLOCK IoStatusBlock, + _Out_writes_bytes_(Length) PVOID FileInformation, + _In_ ULONG Length, + _In_ FILE_INFORMATION_CLASS FileInformationClass, + _In_ BOOLEAN ReturnSingleEntry, + _In_opt_ PUNICODE_STRING FileName, + _In_ BOOLEAN RestartScan +); + +#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L) + +#endif diff --git a/compat/win32/path-utils.c b/compat/win32/path-utils.c index 966ef779b9ca9b..c4fea0301b5ecc 100644 --- a/compat/win32/path-utils.c +++ b/compat/win32/path-utils.c @@ -2,6 +2,9 @@ #include "../../git-compat-util.h" #include "../../environment.h" +#include "../../wrapper.h" +#include "../../strbuf.h" +#include "../../versioncmp.h" int win32_has_dos_drive_prefix(const char *path) { @@ -89,3 +92,199 @@ int win32_fspathcmp(const char *a, const char *b) { return win32_fspathncmp(a, b, (size_t)-1); } + +static int read_at(int fd, char *buffer, size_t offset, size_t size) +{ + if (lseek(fd, offset, SEEK_SET) < 0) { + fprintf(stderr, "could not seek to 0x%x\n", (unsigned int)offset); + return -1; + } + + return read_in_full(fd, buffer, size); +} + +static size_t le16(const char *buffer) +{ + unsigned char *u = (unsigned char *)buffer; + return u[0] | (u[1] << 8); +} + +static size_t le32(const char *buffer) +{ + return le16(buffer) | (le16(buffer + 2) << 16); +} + +/* + * Determine the Go version of a given executable, if it was built with Go. + * + * This recapitulates the logic from + * https://github.com/golang/go/blob/master/src/cmd/go/internal/version/version.go + * (without requiring the user to install `go.exe` to find out). + */ +static ssize_t get_go_version(const char *path, char *go_version, size_t go_version_size) +{ + int fd = open(path, O_RDONLY); + char buffer[1024]; + off_t offset; + size_t num_sections, opt_header_size, i; + char *p = NULL, *q; + ssize_t res = -1; + + if (fd < 0) + return -1; + + if (read_in_full(fd, buffer, 2) < 0) + goto fail; + + /* + * Parse the PE file format, for more details, see + * https://en.wikipedia.org/wiki/Portable_Executable#Layout and + * https://learn.microsoft.com/en-us/windows/win32/debug/pe-format + */ + if (buffer[0] != 'M' || buffer[1] != 'Z') + goto fail; + + if (read_at(fd, buffer, 0x3c, 4) < 0) + goto fail; + + /* Read the `PE\0\0` signature and the COFF file header */ + offset = le32(buffer); + if (read_at(fd, buffer, offset, 24) < 0) + goto fail; + + if (buffer[0] != 'P' || buffer[1] != 'E' || buffer[2] != '\0' || buffer[3] != '\0') + goto fail; + + num_sections = le16(buffer + 6); + opt_header_size = le16(buffer + 20); + offset += 24; /* skip file header */ + + /* + * Validate magic number 0x10b or 0x20b, for full details see + * https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#optional-header-standard-fields-image-only + */ + if (read_at(fd, buffer, offset, 2) < 0 || + ((i = le16(buffer)) != 0x10b && i != 0x20b)) + goto fail; + + offset += opt_header_size; + + for (i = 0; i < num_sections; i++) { + if (read_at(fd, buffer, offset + i * 40, 40) < 0) + goto fail; + + /* + * For full details about the section headers, see + * https://learn.microsoft.com/en-us/windows/win32/debug/pe-format#section-table-section-headers + */ + if ((le32(buffer + 36) /* characteristics */ & ~0x600000) /* IMAGE_SCN_ALIGN_32BYTES */ == + (/* IMAGE_SCN_CNT_INITIALIZED_DATA */ 0x00000040 | + /* IMAGE_SCN_MEM_READ */ 0x40000000 | + /* IMAGE_SCN_MEM_WRITE */ 0x80000000)) { + size_t size = le32(buffer + 16); /* "SizeOfRawData " */ + size_t pointer = le32(buffer + 20); /* "PointerToRawData " */ + + /* + * Skip the section if either size or pointer is 0, see + * https://github.com/golang/go/blob/go1.21.0/src/debug/buildinfo/buildinfo.go#L333 + * for full details. + * + * Merely seeing a non-zero size will not actually do, + * though: he size must be at least `buildInfoSize`, + * i.e. 32, and we expect a UVarint (at least another + * byte) _and_ the bytes representing the string, + * which we expect to start with the letters "go" and + * continue with the Go version number. + */ + if (size < 32 + 1 + 2 + 1 || !pointer) + continue; + + p = malloc(size); + + if (!p || read_at(fd, p, pointer, size) < 0) + goto fail; + + /* + * Look for the build information embedded by Go, see + * https://github.com/golang/go/blob/go1.21.0/src/debug/buildinfo/buildinfo.go#L165-L175 + * for full details. + * + * Note: Go contains code to enforce alignment along a + * 16-byte boundary. In practice, no `.exe` has been + * observed that required any adjustment, therefore + * this here code skips that logic for simplicity. + */ + q = memmem(p, size - 18, "\xff Go buildinf:", 14); + if (!q) + goto fail; + /* + * Decode the build blob. For full details, see + * https://github.com/golang/go/blob/go1.21.0/src/debug/buildinfo/buildinfo.go#L177-L191 + * + * Note: The `endianness` values observed in practice + * were always 2, therefore the complex logic to handle + * any other value is skipped for simplicty. + */ + if ((q[14] == 8 || q[14] == 4) && q[15] == 2) { + /* + * Only handle a Go version string with fewer + * than 128 characters, so the Go UVarint at + * q[32] that indicates the string's length must + * be only one byte (without the high bit set). + */ + if ((q[32] & 0x80) || + !q[32] || + (q + 33 + q[32] - p) > (ssize_t)size || + q[32] + 1 > (ssize_t)go_version_size) + goto fail; + res = q[32]; + memcpy(go_version, q + 33, res); + go_version[res] = '\0'; + break; + } + } + } + +fail: + free(p); + close(fd); + return res; +} + +void win32_warn_about_git_lfs_on_windows7(int exit_code, const char *argv0) +{ + char buffer[128], *git_lfs = NULL; + const char *p; + + /* + * Git LFS v3.5.1 fails with an Access Violation on Windows 7; That + * would usually show up as an exit code 0xc0000005. For some reason + * (probably because at this point, we no longer have the _original_ + * HANDLE that was returned by `CreateProcess()`) we observe other + * values like 0xb00 and 0x2 instead. Since the exact exit code + * seems to be inconsistent, we check for a non-zero exit status. + */ + if (exit_code == 0) + return; + if (GetVersion() >> 16 > 7601) + return; /* Warn only on Windows 7 or older */ + if (!istarts_with(argv0, "git-lfs ") && + strcasecmp(argv0, "git-lfs")) + return; + if (!(git_lfs = locate_in_PATH("git-lfs"))) + return; + if (get_go_version(git_lfs, buffer, sizeof(buffer)) > 0 && + skip_prefix(buffer, "go", &p) && + versioncmp("1.21.0", p) <= 0) + warning("This program was built with Go v%s\n" + "i.e. without support for this Windows version:\n" + "\n\t%s\n" + "\n" + "To work around this, you can download and install a " + "working version from\n" + "\n" + "\thttps://github.com/git-lfs/git-lfs/releases/tag/" + "v3.4.1\n", + p, git_lfs); + free(git_lfs); +} diff --git a/compat/win32/path-utils.h b/compat/win32/path-utils.h index a561c700e75713..a69483c332c1a7 100644 --- a/compat/win32/path-utils.h +++ b/compat/win32/path-utils.h @@ -34,4 +34,7 @@ int win32_fspathcmp(const char *a, const char *b); int win32_fspathncmp(const char *a, const char *b, size_t count); #define fspathncmp win32_fspathncmp +void win32_warn_about_git_lfs_on_windows7(int exit_code, const char *argv0); +#define warn_about_git_lfs_on_windows7 win32_warn_about_git_lfs_on_windows7 + #endif diff --git a/compat/win32/wsl.c b/compat/win32/wsl.c new file mode 100644 index 00000000000000..ab599770138b4e --- /dev/null +++ b/compat/win32/wsl.c @@ -0,0 +1,142 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "../../git-compat-util.h" +#include "../win32.h" +#include "../../repository.h" +#include "config.h" +#include "ntifs.h" +#include "wsl.h" + +int are_wsl_compatible_mode_bits_enabled(void) +{ + /* default to `false` during initialization */ + static const int fallback = 0; + static int enabled = -1; + + if (enabled < 0) { + /* avoid infinite recursion */ + if (!the_repository) + return fallback; + + if (the_repository->config && + the_repository->config->hash_initialized && + repo_config_get_bool(the_repository, "core.wslcompat", &enabled) < 0) + enabled = 0; + } + + return enabled < 0 ? fallback : enabled; +} + +int copy_wsl_mode_bits_from_disk(const wchar_t *wpath, ssize_t wpathlen, + _mode_t *mode) +{ + int ret = -1; + HANDLE h; + if (wpathlen >= 0) { + /* + * It's caller's duty to make sure wpathlen is reasonable so + * it does not overflow. + */ + wchar_t *fn2 = (wchar_t*)alloca((wpathlen + 1) * sizeof(wchar_t)); + memcpy(fn2, wpath, wpathlen * sizeof(wchar_t)); + fn2[wpathlen] = 0; + wpath = fn2; + } + h = CreateFileW(wpath, FILE_READ_EA | SYNCHRONIZE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | + FILE_FLAG_OPEN_REPARSE_POINT, + NULL); + if (h != INVALID_HANDLE_VALUE) { + ret = get_wsl_mode_bits_by_handle(h, mode); + CloseHandle(h); + } + return ret; +} + +#ifndef LX_FILE_METADATA_HAS_UID +#define LX_FILE_METADATA_HAS_UID 0x1 +#define LX_FILE_METADATA_HAS_GID 0x2 +#define LX_FILE_METADATA_HAS_MODE 0x4 +#define LX_FILE_METADATA_HAS_DEVICE_ID 0x8 +#define LX_FILE_CASE_SENSITIVE_DIR 0x10 +typedef struct _FILE_STAT_LX_INFORMATION { + LARGE_INTEGER FileId; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER AllocationSize; + LARGE_INTEGER EndOfFile; + uint32_t FileAttributes; + uint32_t ReparseTag; + uint32_t NumberOfLinks; + ACCESS_MASK EffectiveAccess; + uint32_t LxFlags; + uint32_t LxUid; + uint32_t LxGid; + uint32_t LxMode; + uint32_t LxDeviceIdMajor; + uint32_t LxDeviceIdMinor; +} FILE_STAT_LX_INFORMATION, *PFILE_STAT_LX_INFORMATION; +#endif + +/* + * This struct is extended from the original FILE_FULL_EA_INFORMATION of + * Microsoft Windows. + */ +struct wsl_full_ea_info_t { + uint32_t NextEntryOffset; + uint8_t Flags; + uint8_t EaNameLength; + uint16_t EaValueLength; + char EaName[7]; + char EaValue[4]; + char Padding[1]; +}; + +enum { + FileStatLxInformation = 70, +}; +__declspec(dllimport) NTSTATUS WINAPI + NtQueryInformationFile(HANDLE FileHandle, + PIO_STATUS_BLOCK IoStatusBlock, + PVOID FileInformation, ULONG Length, + uint32_t FileInformationClass); +__declspec(dllimport) NTSTATUS WINAPI + NtSetInformationFile(HANDLE FileHandle, PIO_STATUS_BLOCK IoStatusBlock, + PVOID FileInformation, ULONG Length, + uint32_t FileInformationClass); +__declspec(dllimport) NTSTATUS WINAPI + NtSetEaFile(HANDLE FileHandle, PIO_STATUS_BLOCK IoStatusBlock, + PVOID EaBuffer, ULONG EaBufferSize); + +int set_wsl_mode_bits_by_handle(HANDLE h, _mode_t mode) +{ + uint32_t value = mode; + struct wsl_full_ea_info_t ea_info; + IO_STATUS_BLOCK iob; + /* mode should be valid to make WSL happy */ + assert(S_ISREG(mode) || S_ISDIR(mode)); + ea_info.NextEntryOffset = 0; + ea_info.Flags = 0; + ea_info.EaNameLength = 6; + ea_info.EaValueLength = sizeof(value); /* 4 */ + strlcpy(ea_info.EaName, "$LXMOD", sizeof(ea_info.EaName)); + memcpy(ea_info.EaValue, &value, sizeof(value)); + ea_info.Padding[0] = 0; + return NtSetEaFile(h, &iob, &ea_info, sizeof(ea_info)); +} + +int get_wsl_mode_bits_by_handle(HANDLE h, _mode_t *mode) +{ + FILE_STAT_LX_INFORMATION fxi; + IO_STATUS_BLOCK iob; + if (NtQueryInformationFile(h, &iob, &fxi, sizeof(fxi), + FileStatLxInformation) == 0) { + if (fxi.LxFlags & LX_FILE_METADATA_HAS_MODE) + *mode = (_mode_t)fxi.LxMode; + return 0; + } + return -1; +} diff --git a/compat/win32/wsl.h b/compat/win32/wsl.h new file mode 100644 index 00000000000000..1f5ad7e67a4fc2 --- /dev/null +++ b/compat/win32/wsl.h @@ -0,0 +1,12 @@ +#ifndef COMPAT_WIN32_WSL_H +#define COMPAT_WIN32_WSL_H + +int are_wsl_compatible_mode_bits_enabled(void); + +int copy_wsl_mode_bits_from_disk(const wchar_t *wpath, ssize_t wpathlen, + _mode_t *mode); + +int get_wsl_mode_bits_by_handle(HANDLE h, _mode_t *mode); +int set_wsl_mode_bits_by_handle(HANDLE h, _mode_t mode); + +#endif diff --git a/compat/winansi.c b/compat/winansi.c index 3ce190093901b4..2037e3d87d1c48 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -546,6 +546,9 @@ static void detect_msys_tty(int fd) if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation, buffer, sizeof(buffer) - 2, &result))) return; + if (result < sizeof(*nameinfo) || !nameinfo->Name.Buffer || + !nameinfo->Name.Length) + return; name = nameinfo->Name.Buffer; name[nameinfo->Name.Length / sizeof(*name)] = 0; @@ -564,6 +567,49 @@ static void detect_msys_tty(int fd) #endif +static HANDLE std_console_handle; +static DWORD std_console_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING; +static UINT std_console_code_page = CP_UTF8; + +static void reset_std_console(void) +{ + if (std_console_mode != ENABLE_VIRTUAL_TERMINAL_PROCESSING) + SetConsoleMode(std_console_handle, std_console_mode); + if (std_console_code_page != CP_UTF8) + SetConsoleOutputCP(std_console_code_page); +} + +static int enable_virtual_processing(void) +{ + std_console_handle = GetStdHandle(STD_OUTPUT_HANDLE); + if (std_console_handle == INVALID_HANDLE_VALUE || + !GetConsoleMode(std_console_handle, &std_console_mode)) { + std_console_handle = GetStdHandle(STD_ERROR_HANDLE); + if (std_console_handle == INVALID_HANDLE_VALUE || + !GetConsoleMode(std_console_handle, &std_console_mode)) + return 0; + } + + std_console_code_page = GetConsoleOutputCP(); + if (std_console_code_page != CP_UTF8) + SetConsoleOutputCP(CP_UTF8); + if (!std_console_code_page) + std_console_code_page = CP_UTF8; + + atexit(reset_std_console); + + if (std_console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) + return 1; + + if (!SetConsoleMode(std_console_handle, + std_console_mode | + ENABLE_PROCESSED_OUTPUT | + ENABLE_VIRTUAL_TERMINAL_PROCESSING)) + return 0; + + return 1; +} + /* * Wrapper for isatty(). Most calls in the main git code * call isatty(1 or 2) to see if the instance is interactive @@ -602,6 +648,9 @@ void winansi_init(void) return; } + if (enable_virtual_processing()) + return; + /* create a named pipe to communicate with the console thread */ if (swprintf(name, ARRAY_SIZE(name) - 1, L"\\\\.\\pipe\\winansi%lu", GetCurrentProcessId()) < 0) diff --git a/config.mak.dev b/config.mak.dev index eecb12c1111ff4..7201fc0fbe2f4d 100644 --- a/config.mak.dev +++ b/config.mak.dev @@ -21,12 +21,15 @@ endif endif ifneq ($(uname_S),FreeBSD) -ifneq ($(or $(filter gcc6,$(COMPILER_FEATURES)),$(filter clang7,$(COMPILER_FEATURES))),) +ifneq ($(filter gcc6,$(COMPILER_FEATURES)),) DEVELOPER_CFLAGS += -std=gnu99 endif else # FreeBSD cannot limit to C99 because its system headers unconditionally # rely on C11 features. +# +# Clang cannot limit to C99 when using glibc 2.43 because its system headers +# depend on the _Generic C11 feature. This works with GCC though. endif DEVELOPER_CFLAGS += -Wdeclaration-after-statement diff --git a/config.mak.uname b/config.mak.uname index f9a5ad97209dfd..1574fe4eba48c9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -452,14 +452,8 @@ ifeq ($(uname_S),Windows) GIT_VERSION := $(GIT_VERSION).MSVC pathsep = ; # Assume that this is built in Git for Windows' SDK - ifeq (MINGW32,$(MSYSTEM)) - prefix = /mingw32 - else - ifeq (CLANGARM64,$(MSYSTEM)) - prefix = /clangarm64 - else - prefix = /mingw64 - endif + ifneq (,$(MSYSTEM)) + prefix = $(MINGW_PREFIX) endif # Prepend MSVC 64-bit tool-chain to PATH. # @@ -467,7 +461,7 @@ ifeq ($(uname_S),Windows) # link.exe next to, and required by, cl.exe, we have to prepend this # onto the existing $PATH. # - SANE_TOOL_PATH ?= $(msvc_bin_dir_msys) + SANE_TOOL_PATH ?= $(msvc_bin_dir_msys):$(sdk_ver_bin_dir_msys) HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NEEDS_CRYPTO_WITH_SSL = YesPlease @@ -511,7 +505,8 @@ ifeq ($(uname_S),Windows) NO_POSIX_GOODIES = UnfortunatelyYes NATIVE_CRLF = YesPlease DEFAULT_HELP_FORMAT = html -ifeq (/mingw64,$(subst 32,64,$(subst clangarm,mingw,$(prefix)))) + SKIP_DASHED_BUILT_INS = YabbaDabbaDoo +ifneq (,$(MINGW_PREFIX)) # Move system config into top-level /etc/ ETC_GITCONFIG = ../etc/gitconfig ETC_GITATTRIBUTES = ../etc/gitattributes @@ -526,14 +521,18 @@ endif compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/trace2_win32_process_info.o \ - compat/win32/dirent.o - COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" - BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE + compat/win32/dirent.o compat/win32/fscache.o compat/win32/wsl.o + COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \ + -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \ + -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" + BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO # invalidcontinue.obj allows Git's source code to close the same file # handle twice, or to access the osfhandle of an already-closed stdout # See https://msdn.microsoft.com/en-us/library/ms235330.aspx EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib + GITLIBS += git.res PTHREAD_LIBS = + RC = compat/vcbuild/scripts/rc.pl lib = BASIC_CFLAGS += $(vcpkg_inc) $(sdk_includes) $(msvc_includes) ifndef DEBUG @@ -703,6 +702,7 @@ ifeq ($(uname_S),MINGW) FSMONITOR_DAEMON_BACKEND = win32 FSMONITOR_OS_SETTINGS = win32 + SKIP_DASHED_BUILT_INS = YabbaDabbaDoo RUNTIME_PREFIX = YesPlease HAVE_WPGMPTR = YesWeDo NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease @@ -717,7 +717,8 @@ ifeq ($(uname_S),MINGW) DEFAULT_HELP_FORMAT = html HAVE_PLATFORM_PROCINFO = YesPlease CSPRNG_METHOD = rtlgenrandom - BASIC_LDFLAGS += -municode + BASIC_LDFLAGS += -municode -Wl,--tsaware + LAZYLOAD_LIBCURL = YesDoThatPlease COMPAT_CFLAGS += -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ @@ -725,7 +726,7 @@ ifeq ($(uname_S),MINGW) compat/win32/flush.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/fscache.o compat/win32/wsl.o BASIC_CFLAGS += -DWIN32 EXTLIBS += -lws2_32 GITLIBS += git.res @@ -741,26 +742,25 @@ ifeq ($(uname_S),MINGW) ifneq (,$(findstring -O,$(filter-out -O0 -Og,$(CFLAGS)))) BASIC_LDFLAGS += -Wl,--dynamicbase endif - ifeq (MINGW32,$(MSYSTEM)) - prefix = /mingw32 - HOST_CPU = i686 - BASIC_LDFLAGS += -Wl,--pic-executable,-e,_mainCRTStartup - endif - ifeq (MINGW64,$(MSYSTEM)) - prefix = /mingw64 - HOST_CPU = x86_64 - BASIC_LDFLAGS += -Wl,--pic-executable,-e,mainCRTStartup - else ifeq (CLANGARM64,$(MSYSTEM)) - prefix = /clangarm64 - HOST_CPU = aarch64 - BASIC_LDFLAGS += -Wl,--pic-executable,-e,mainCRTStartup - else - COMPAT_CFLAGS += -D_USE_32BIT_TIME_T - BASIC_LDFLAGS += -Wl,--large-address-aware + ifneq (,$(MSYSTEM)) + ifeq ($(MINGW_PREFIX),$(filter-out /%,$(MINGW_PREFIX))) + # Override if empty or does not start with a slash + MINGW_PREFIX := /$(shell echo '$(MSYSTEM)' | tr A-Z a-z) + endif + prefix = $(MINGW_PREFIX) + HOST_CPU = $(patsubst %-w64-mingw32,%,$(MINGW_CHOST)) + BASIC_LDFLAGS += -Wl,--pic-executable + COMPAT_CFLAGS += -DDETECT_MSYS_TTY \ + -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" \ + -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" + ifeq (MINGW32,$(MSYSTEM)) + BASIC_LDFLAGS += -Wl,--large-address-aware + endif + # Move system config into top-level /etc/ + ETC_GITCONFIG = ../etc/gitconfig + ETC_GITATTRIBUTES = ../etc/gitattributes endif - CC = gcc - COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -DDETECT_MSYS_TTY \ - -fstack-protector-strong + COMPAT_CFLAGS += -D__USE_MINGW_ANSI_STDIO=0 -fstack-protector-strong EXTLIBS += -lntdll EXTRA_PROGRAMS += headless-git$X INSTALL = /bin/install @@ -768,11 +768,7 @@ ifeq ($(uname_S),MINGW) HAVE_LIBCHARSET_H = YesPlease USE_GETTEXT_SCHEME = fallthrough USE_LIBPCRE = YesPlease - ifeq (/mingw64,$(subst 32,64,$(subst clangarm,mingw,$(prefix)))) - # Move system config into top-level /etc/ - ETC_GITCONFIG = ../etc/gitconfig - ETC_GITATTRIBUTES = ../etc/gitattributes - endif + NO_PYTHON = endif ifeq ($(uname_S),QNX) COMPAT_CFLAGS += -DSA_RESTART=0 diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a57c4b464fa456..9077b187e5e635 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -14,6 +14,11 @@ Note: Visual Studio also has the option of opening `CMakeLists.txt` directly; Using this option, Visual Studio will not find the source code, though, therefore the `File>Open>Folder...` option is preferred. +Visual Studio does not produce a .sln solution file nor the .vcxproj files +that may be required by VS extension tools. + +To generate the .sln/.vcxproj files run CMake manually, as described below. + Instructions to run CMake manually: mkdir -p contrib/buildsystems/out @@ -22,7 +27,7 @@ Instructions to run CMake manually: This will build the git binaries in contrib/buildsystems/out directory (our top-level .gitignore file knows to ignore contents of -this directory). +this directory). The project .sln and .vcxproj files are also generated. Possible build configurations(-DCMAKE_BUILD_TYPE) with corresponding compiler flags @@ -35,17 +40,16 @@ empty(default) : NOTE: -DCMAKE_BUILD_TYPE is optional. For multi-config generators like Visual Studio this option is ignored -This process generates a Makefile(Linux/*BSD/MacOS) , Visual Studio solution(Windows) by default. +This process generates a Makefile(Linux/*BSD/MacOS), Visual Studio solution(Windows) by default. Run `make` to build Git on Linux/*BSD/MacOS. Open git.sln on Windows and build Git. -NOTE: By default CMake uses Makefile as the build tool on Linux and Visual Studio in Windows, -to use another tool say `ninja` add this to the command line when configuring. -`-G Ninja` - NOTE: By default CMake will install vcpkg locally to your source tree on configuration, to avoid this, add `-DNO_VCPKG=TRUE` to the command line when configuring. +The Visual Studio default generator changed in v16.6 from its Visual Studio +implemenation to `Ninja` This required changes to many CMake scripts. + ]] cmake_minimum_required(VERSION 3.14) @@ -59,15 +63,29 @@ endif() if(NOT DEFINED CMAKE_EXPORT_COMPILE_COMMANDS) set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) + message("settting CMAKE_EXPORT_COMPILE_COMMANDS: ${CMAKE_EXPORT_COMPILE_COMMANDS}") endif() if(USE_VCPKG) set(VCPKG_DIR "${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg") + message("WIN32: ${WIN32}") # show its underlying text values + message("VCPKG_DIR: ${VCPKG_DIR}") + message("VCPKG_ARCH: ${VCPKG_ARCH}") # maybe unset + message("MSVC: ${MSVC}") + message("CMAKE_GENERATOR: ${CMAKE_GENERATOR}") + message("CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") + message("CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}") + message("CMAKE_EXPORT_COMPILE_COMMANDS: ${CMAKE_EXPORT_COMPILE_COMMANDS}") + message("ENV(CMAKE_EXPORT_COMPILE_COMMANDS): $ENV{CMAKE_EXPORT_COMPILE_COMMANDS}") if(NOT EXISTS ${VCPKG_DIR}) message("Initializing vcpkg and building the Git's dependencies (this will take a while...)") - execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat) + execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat ${VCPKG_ARCH}) + endif() + if(NOT EXISTS ${VCPKG_ARCH}) + message("VCPKG_ARCH: unset, using 'x64-windows'") + set(VCPKG_ARCH "x64-windows") # default from vcpkg_install.bat endif() - list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/x64-windows") + list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/${VCPKG_ARCH}") # In the vcpkg edition, we need this to be able to link to libcurl set(CURL_NO_CURL_CMAKE ON) @@ -208,11 +226,19 @@ if(CMAKE_C_COMPILER_ID STREQUAL "MSVC") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}) add_compile_options(/MP /std:c11) + add_link_options(/MANIFEST:NO) endif() #default behaviour include_directories(${CMAKE_SOURCE_DIR}) -add_compile_definitions(GIT_HOST_CPU="${CMAKE_SYSTEM_PROCESSOR}") + +# When cross-compiling, define HOST_CPU as the canonical name of the CPU on +# which the built Git will run (for instance "x86_64"). +if(NOT HOST_CPU) + add_compile_definitions(GIT_HOST_CPU="${CMAKE_SYSTEM_PROCESSOR}") +else() + add_compile_definitions(GIT_HOST_CPU="${HOST_CPU}") +endif() add_compile_definitions(SHA256_BLK INTERNAL_QSORT RUNTIME_PREFIX) add_compile_definitions(NO_OPENSSL SHA1_DC SHA1DC_NO_STANDARD_INCLUDES SHA1DC_INIT_SAFE_HASH_DEFAULT=0 @@ -256,7 +282,14 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") _CONSOLE DETECT_MSYS_TTY STRIP_EXTENSION=".exe" NO_SYMLINK_HEAD UNRELIABLE_FSTAT NOGDI OBJECT_CREATION_MODE=1 __USE_MINGW_ANSI_STDIO=0 OVERRIDE_STRDUP MMAP_PREVENTS_DELETE USE_WIN32_MMAP - HAVE_WPGMPTR ENSURE_MSYSTEM_IS_SET HAVE_RTLGENRANDOM) + HAVE_WPGMPTR HAVE_RTLGENRANDOM) + if(CMAKE_GENERATOR_PLATFORM STREQUAL "x64") + add_compile_definitions(ENSURE_MSYSTEM_IS_SET="MINGW64" MINGW_PREFIX="mingw64") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "arm64") + add_compile_definitions(ENSURE_MSYSTEM_IS_SET="CLANGARM64" MINGW_PREFIX="clangarm64") + elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "x86") + add_compile_definitions(ENSURE_MSYSTEM_IS_SET="MINGW32" MINGW_PREFIX="mingw32") + endif() list(APPEND compat_SOURCES compat/mingw.c compat/winansi.c @@ -267,7 +300,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") compat/win32/syslog.c compat/win32/trace2_win32_process_info.c compat/win32/dirent.c - compat/strdup.c) + compat/win32/wsl.c + compat/strdup.c + compat/win32/fscache.c) set(NO_UNIX_SOCKETS 1) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") @@ -733,6 +768,7 @@ if(WIN32) endif() add_executable(headless-git ${CMAKE_SOURCE_DIR}/compat/win32/headless.c) + list(APPEND PROGRAMS_BUILT headless-git) if(CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_C_COMPILER_ID STREQUAL "Clang") target_link_options(headless-git PUBLIC -municode -Wl,-subsystem,windows) elseif(CMAKE_C_COMPILER_ID STREQUAL "MSVC") @@ -933,7 +969,7 @@ list(TRANSFORM git_perl_scripts PREPEND "${CMAKE_BINARY_DIR}/") #install foreach(program ${PROGRAMS_BUILT}) -if(program MATCHES "^(git|git-shell|scalar)$") +if(program MATCHES "^(git|git-shell|headless-git|scalar)$") install(TARGETS ${program} RUNTIME DESTINATION bin) else() @@ -1201,7 +1237,7 @@ string(REPLACE "@USE_LIBPCRE2@" "" git_build_options "${git_build_options}") string(REPLACE "@WITH_BREAKING_CHANGES@" "" git_build_options "${git_build_options}") string(REPLACE "@X@" "${EXE_EXTENSION}" git_build_options "${git_build_options}") if(USE_VCPKG) - string(APPEND git_build_options "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/x64-windows/bin\"\n") + string(APPEND git_build_options "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/${VCPKG_ARCH}/bin\"\n") endif() file(WRITE ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS ${git_build_options}) diff --git a/contrib/subtree/Makefile b/contrib/subtree/Makefile index c0c9f21cb78022..dab2dfc08ee222 100644 --- a/contrib/subtree/Makefile +++ b/contrib/subtree/Makefile @@ -95,7 +95,7 @@ $(GIT_SUBTREE_TEST): $(GIT_SUBTREE) cp $< $@ test: $(GIT_SUBTREE_TEST) - $(MAKE) -C t/ test + $(MAKE) -C t/ all clean: $(RM) $(GIT_SUBTREE) diff --git a/credential.c b/credential.c index 2594c0c4229ba0..af964189363b28 100644 --- a/credential.c +++ b/credential.c @@ -360,6 +360,9 @@ int credential_read(struct credential *c, FILE *fp, credential_set_capability(&c->capa_authtype, op_type); else if (!strcmp(value, "state")) credential_set_capability(&c->capa_state, op_type); + } else if (!strcmp(key, "ntlm")) { + if (!strcmp(value, "allow")) + c->ntlm_allow = 1; } else if (!strcmp(key, "continue")) { c->multistage = !!git_config_bool("continue", value); } else if (!strcmp(key, "password_expiry_utc")) { @@ -420,6 +423,8 @@ void credential_write(const struct credential *c, FILE *fp, if (c->ephemeral) credential_write_item(c, fp, "ephemeral", "1", 0); } + if (c->ntlm_suppressed) + credential_write_item(c, fp, "ntlm", "suppressed", 0); credential_write_item(c, fp, "protocol", c->protocol, 1); credential_write_item(c, fp, "host", c->host, 1); credential_write_item(c, fp, "path", c->path, 0); diff --git a/credential.h b/credential.h index c78b72d110eaac..95244d5375dfe9 100644 --- a/credential.h +++ b/credential.h @@ -177,6 +177,9 @@ struct credential { struct credential_capability capa_authtype; struct credential_capability capa_state; + unsigned ntlm_suppressed:1, + ntlm_allow:1; + char *username; char *password; char *credential; diff --git a/dir.c b/dir.c index 33c81c256ee925..5842c777cfdfc8 100644 --- a/dir.c +++ b/dir.c @@ -1156,16 +1156,64 @@ static int add_patterns(const char *fname, const char *base, int baselen, size_t size = 0; char *buf; - if (flags & PATTERN_NOFOLLOW) - fd = open_nofollow(fname, O_RDONLY); - else - fd = open(fname, O_RDONLY); - - if (fd < 0 || fstat(fd, &st) < 0) { - if (fd < 0) - warn_on_fopen_errors(fname); + /* + * A performance optimization for status. + * + * During a status scan, git looks in each directory for a .gitignore + * file before scanning the directory. Since .gitignore files are not + * that common, we can waste a lot of time looking for files that are + * not there. Fortunately, the fscache already knows if the directory + * contains a .gitignore file, since it has already read the directory + * and it already has the stat-data. + * + * If the fscache is enabled, use the fscache-lstat() interlude to see + * if the file exists (in the fscache hash maps) before trying to open() + * it. + * + * This causes problem when the .gitignore file is a symlink, because + * we call lstat() rather than stat() on the symlnk and the resulting + * stat-data is for the symlink itself rather than the target file. + * We CANNOT use stat() here because the fscache DOES NOT install an + * interlude for stat() and mingw_stat() always calls "open-fstat-close" + * on the file and defeats the purpose of the optimization here. Since + * symlinks are even more rare than .gitignore files, we force a fstat() + * after our open() to get stat-data for the target file. + * + * Since `clang`'s `-Wunreachable-code` mode is clever, it would figure + * out that on non-Windows platforms, this `lstat()` is unreachable. + * We do want to keep the conditional block for the sake of Windows, + * though, so let's use the `NOT_CONSTANT()` trick to suppress that error. + */ + if (NOT_CONSTANT(is_fscache_enabled(fname))) { + if (lstat(fname, &st) < 0) { + fd = -1; + } else { + fd = open(fname, O_RDONLY); + if (fd < 0) + warn_on_fopen_errors(fname); + else if (S_ISLNK(st.st_mode) && fstat(fd, &st) < 0) { + warn_on_fopen_errors(fname); + close(fd); + fd = -1; + } + } + } else { + if (flags & PATTERN_NOFOLLOW) + fd = open_nofollow(fname, O_RDONLY); else - close(fd); + fd = open(fname, O_RDONLY); + + if (fd < 0 || fstat(fd, &st) < 0) { + if (fd < 0) + warn_on_fopen_errors(fname); + else { + close(fd); + fd = -1; + } + } + } + + if (fd < 0) { if (!istate) return -1; r = read_skip_worktree_file_from_index(istate, fname, @@ -3411,6 +3459,13 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) return 0; } + if (is_mount_point(path)) { + /* Do not descend and nuke a mount point or junction. */ + if (kept_up) + *kept_up = 1; + return 0; + } + flag &= ~REMOVE_DIR_KEEP_TOPLEVEL; dir = opendir(path->buf); if (!dir) { diff --git a/editor.c b/editor.c index fd174e6a034f1c..f6d960c6f30782 100644 --- a/editor.c +++ b/editor.c @@ -13,6 +13,7 @@ #include "strvec.h" #include "run-command.h" #include "sigchain.h" +#include "compat/terminal.h" #ifndef DEFAULT_EDITOR #define DEFAULT_EDITOR "vi" @@ -64,6 +65,7 @@ static int launch_specified_editor(const char *editor, const char *path, return error("Terminal is dumb, but EDITOR unset"); if (strcmp(editor, ":")) { + int save_and_restore_term = !strcmp(editor, "vi") || !strcmp(editor, "vim"); struct strbuf realpath = STRBUF_INIT; struct child_process p = CHILD_PROCESS_INIT; int ret, sig; @@ -92,7 +94,11 @@ static int launch_specified_editor(const char *editor, const char *path, strvec_pushv(&p.env, (const char **)env); p.use_shell = 1; p.trace2_child_class = "editor"; + if (save_and_restore_term) + save_and_restore_term = !save_term(1); if (start_command(&p) < 0) { + if (save_and_restore_term) + restore_term(); strbuf_release(&realpath); return error("unable to start editor '%s'", editor); } @@ -100,6 +106,8 @@ static int launch_specified_editor(const char *editor, const char *path, sigchain_push(SIGINT, SIG_IGN); sigchain_push(SIGQUIT, SIG_IGN); ret = finish_command(&p); + if (save_and_restore_term) + restore_term(); strbuf_release(&realpath); sig = ret - 128; sigchain_pop(SIGINT); diff --git a/entry.c b/entry.c index 7817aee362ed9e..b299e3f1071ff6 100644 --- a/entry.c +++ b/entry.c @@ -324,7 +324,7 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca if (!has_symlinks || to_tempfile) goto write_file_entry; - ret = symlink(new_blob, path); + ret = create_symlink(state->istate, new_blob, path); free(new_blob); if (ret) return error_errno("unable to create symlink %s", path); @@ -411,6 +411,9 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca } finish: + /* Flush cached lstat in fscache after writing to disk. */ + flush_fscache(); + if (state->refresh_cache) { if (!fstat_done && lstat(ce->name, &st) < 0) return error_errno("unable to stat just-written file %s", diff --git a/fetch-pack.c b/fetch-pack.c index 120e01f3cf2674..a9fa07c2bb2866 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -802,6 +802,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, save_commit_buffer = 0; trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL); + enable_fscache(0); for (ref = *refs; ref; ref = ref->next) { struct commit *commit; @@ -826,6 +827,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, if (!cutoff || cutoff < commit->date) cutoff = commit->date; } + disable_fscache(); trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL); /* diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index a6587a8972b184..b4c29f44a27827 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -5,6 +5,7 @@ #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" #include "fsmonitor-path-utils.h" +#include "advice.h" /* * We keep this structure definition private and have getters @@ -100,6 +101,31 @@ static struct fsmonitor_settings *alloc_settings(void) return s; } +static int check_deprecated_builtin_config(struct repository *r) +{ + int core_use_builtin_fsmonitor = 0; + + /* + * If 'core.useBuiltinFSMonitor' is set, print a deprecation warning + * suggesting the use of 'core.fsmonitor' instead. If the config is + * set to true, set the appropriate mode and return 1 indicating that + * the check resulted the config being set by this (deprecated) setting. + */ + if(!repo_config_get_bool(r, "core.useBuiltinFSMonitor", &core_use_builtin_fsmonitor) && + core_use_builtin_fsmonitor) { + if (!git_env_bool("GIT_SUPPRESS_USEBUILTINFSMONITOR_ADVICE", 0)) { + advise_if_enabled(ADVICE_USE_CORE_FSMONITOR_CONFIG, + _("core.useBuiltinFSMonitor=true is deprecated;" + "please set core.fsmonitor=true instead")); + setenv("GIT_SUPPRESS_USEBUILTINFSMONITOR_ADVICE", "1", 1); + } + fsm_settings__set_ipc(r); + return 1; + } + + return 0; +} + static void lookup_fsmonitor_settings(struct repository *r) { const char *const_str; @@ -126,12 +152,16 @@ static void lookup_fsmonitor_settings(struct repository *r) return; case 1: /* config value was unset */ + if (check_deprecated_builtin_config(r)) + return; + const_str = getenv("GIT_TEST_FSMONITOR"); break; case -1: /* config value set to an arbitrary string */ - if (repo_config_get_pathname(r, "core.fsmonitor", &to_free)) - return; /* should not happen */ + if (check_deprecated_builtin_config(r) || + repo_config_get_pathname(r, "core.fsmonitor", &to_free)) + return; const_str = to_free; break; diff --git a/git-compat-util.h b/git-compat-util.h index 88097764078538..0fb51b936bf05b 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -158,9 +158,11 @@ static inline int is_xplatform_dir_sep(int c) /* pull in Windows compatibility stuff */ #include "compat/win32/path-utils.h" #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/win32/path-utils.h" #include "compat/msvc.h" +#include "compat/win32/fscache.h" #endif /* used on Mac OS X */ @@ -261,6 +263,13 @@ static inline int git_offset_1st_component(const char *path) #define fspathncmp git_fspathncmp #endif +#ifndef warn_about_git_lfs_on_windows7 +static inline void warn_about_git_lfs_on_windows7(int exit_code UNUSED, + const char *argv0 UNUSED) +{ +} +#endif + #ifndef is_valid_path #define is_valid_path(path) 1 #endif @@ -346,10 +355,28 @@ static inline int git_has_dir_sep(const char *path) #define has_dir_sep(path) git_has_dir_sep(path) #endif +#ifndef is_mount_point +#define is_mount_point is_mount_point_via_stat +#endif + +#ifndef create_symlink +struct index_state; +static inline int git_create_symlink(struct index_state *index UNUSED, + const char *target, const char *link) +{ + return symlink(target, link); +} +#define create_symlink git_create_symlink +#endif + #ifndef query_user_email #define query_user_email() NULL #endif +#ifndef platform_strbuf_realpath +#define platform_strbuf_realpath(resolved, path) NULL +#endif + #ifdef __TANDEM #include #include @@ -597,17 +624,23 @@ static inline bool strip_suffix(const char *str, const char *suffix, * the stack overflow can occur. */ #define DEFAULT_MAX_ALLOWED_TREE_DEPTH 512 -#elif defined(GIT_WINDOWS_NATIVE) && defined(__clang__) && defined(__aarch64__) +#elif defined(GIT_WINDOWS_NATIVE) && defined(__clang__) /* - * Similar to Visual C, it seems that on Windows/ARM64 the clang-based - * builds have a smaller stack space available. When running out of - * that stack space, a `STATUS_STACK_OVERFLOW` is produced. When the + * Similar to Visual C, it seems that clang-based builds on Windows + * have a smaller stack space available. When running out of that + * stack space, a `STATUS_STACK_OVERFLOW` is produced. When the * Git command was run from an MSYS2 Bash, this unfortunately results * in an exit code 127. Let's prevent that by lowering the maximal - * tree depth; This value seems to be low enough. + * tree depth; Unfortunately, it seems that the exact limit differs + * for aarch64 vs x86_64, and the difference is too large to simply + * use a single limit. */ +#if defined(__aarch64__) #define DEFAULT_MAX_ALLOWED_TREE_DEPTH 1280 #else +#define DEFAULT_MAX_ALLOWED_TREE_DEPTH 1152 +#endif +#else #define DEFAULT_MAX_ALLOWED_TREE_DEPTH 2048 #endif @@ -1064,6 +1097,45 @@ static inline int is_missing_file_error(int errno_) return (errno_ == ENOENT || errno_ == ENOTDIR); } +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ + + /* opaque fscache structure */ +struct fscache; + +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + +#ifndef disable_fscache +#define disable_fscache() /* noop */ +#endif + +#ifndef is_fscache_enabled +#define is_fscache_enabled(path) (0) +#endif + +#ifndef flush_fscache +#define flush_fscache() /* noop */ +#endif + +#ifndef getcache_fscache +#define getcache_fscache() (NULL) /* noop */ +#endif + +#ifndef merge_fscache +#define merge_fscache(dest) /* noop */ +#endif + int cmd_main(int, const char **); /* diff --git a/git-curl-compat.h b/git-curl-compat.h index dccdd4d6e54158..5c8ceb076adea2 100644 --- a/git-curl-compat.h +++ b/git-curl-compat.h @@ -45,6 +45,14 @@ #define GIT_CURL_HAVE_CURLINFO_RETRY_AFTER 1 #endif +/** + * CURLSSLOPT_AUTO_CLIENT_CERT was added in 7.77.0, released in May + * 2021. + */ +#if LIBCURL_VERSION_NUM >= 0x074d00 +#define GIT_CURL_HAVE_CURLSSLOPT_AUTO_CLIENT_CERT +#endif + /** * CURLOPT_PROTOCOLS_STR and CURLOPT_REDIR_PROTOCOLS_STR were added in 7.85.0, * released in August 2022. diff --git a/git-gui/git-gui--askyesno.sh b/git-gui/git-gui--askyesno.sh index 142d1bc3de229b..e431f86a8e16ae 100755 --- a/git-gui/git-gui--askyesno.sh +++ b/git-gui/git-gui--askyesno.sh @@ -29,8 +29,8 @@ if {$argc < 1} { } ${NS}::frame .t -${NS}::label .t.m -text $prompt -justify center -width 40 -.t.m configure -wraplength 400 +${NS}::label .t.m -text $prompt -justify center -width 400px +.t.m configure -wraplength 400px pack .t.m -side top -fill x -padx 20 -pady 20 -expand 1 pack .t -side top -fill x -ipadx 20 -ipady 20 -expand 1 @@ -59,5 +59,17 @@ if {$::tcl_platform(platform) eq {windows}} { } } +if {$::tcl_platform(platform) eq {windows}} { + set icopath [file dirname [file normalize $argv0]] + if {[file tail $icopath] eq {git-core}} { + set icopath [file dirname $icopath] + } + set icopath [file dirname $icopath] + set icopath [file join $icopath share git git-for-windows.ico] + if {[file exists $icopath]} { + wm iconbitmap . -default $icopath + } +} + wm title . $title tk::PlaceWindow . diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 23fe76e498bd17..799b564b926d0f 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -1934,6 +1934,7 @@ set all_icons(U$ui_index) file_merge set all_icons(T$ui_index) file_statechange set all_icons(_$ui_workdir) file_plain +set all_icons(A$ui_workdir) file_plain set all_icons(M$ui_workdir) file_mod set all_icons(D$ui_workdir) file_question set all_icons(U$ui_workdir) file_merge @@ -1960,6 +1961,7 @@ foreach i { {A_ {mc "Staged for commit"}} {AM {mc "Portions staged for commit"}} {AD {mc "Staged for commit, missing"}} + {AA {mc "Intended to be added"}} {_D {mc "Missing"}} {D_ {mc "Staged for removal"}} diff --git a/git-gui/lib/diff.tcl b/git-gui/lib/diff.tcl index 8be1a613fbe01f..d25a9bbdc4abde 100644 --- a/git-gui/lib/diff.tcl +++ b/git-gui/lib/diff.tcl @@ -556,7 +556,8 @@ proc apply_or_revert_hunk {x y revert} { if {$current_diff_side eq $ui_index} { set failed_msg [mc "Failed to unstage selected hunk."] lappend apply_cmd --reverse --cached - if {[string index $mi 0] ne {M}} { + set file_state [string index $mi 0] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -569,7 +570,8 @@ proc apply_or_revert_hunk {x y revert} { lappend apply_cmd --cached } - if {[string index $mi 1] ne {M}} { + set file_state [string index $mi 1] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -661,7 +663,8 @@ proc apply_or_revert_range_or_line {x y revert} { set failed_msg [mc "Failed to unstage selected line."] set to_context {+} lappend apply_cmd --reverse --cached - if {[string index $mi 0] ne {M}} { + set file_state [string index $mi 0] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -676,7 +679,8 @@ proc apply_or_revert_range_or_line {x y revert} { lappend apply_cmd --cached } - if {[string index $mi 1] ne {M}} { + set file_state [string index $mi 1] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } diff --git a/git-sh-setup.sh b/git-sh-setup.sh index 19aef72ec25530..c51ad34148ccf3 100644 --- a/git-sh-setup.sh +++ b/git-sh-setup.sh @@ -292,17 +292,30 @@ create_virtual_base() { # Platform specific tweaks to work around some commands case $(uname -s) in *MINGW*) - # Windows has its own (incompatible) sort and find - sort () { - /usr/bin/sort "$@" - } - find () { - /usr/bin/find "$@" - } - # git sees Windows-style pwd - pwd () { - builtin pwd -W - } + if test -x /usr/bin/sort + then + # Windows has its own (incompatible) sort; override + sort () { + /usr/bin/sort "$@" + } + fi + if test -x /usr/bin/find + then + # Windows has its own (incompatible) find; override + find () { + /usr/bin/find "$@" + } + fi + # On Windows, Git wants Windows paths. But /usr/bin/pwd spits out + # Unix-style paths. At least in Bash, we have a builtin pwd that + # understands the -W option to force "mixed" paths, i.e. with drive + # prefix but still with forward slashes. Let's use that, if available. + if type builtin >/dev/null 2>&1 + then + pwd () { + builtin pwd -W + } + fi is_absolute_path () { case "$1" in [/\\]* | [A-Za-z]:*) diff --git a/git.c b/git.c index 36f08891ef5476..632fdfd86c6385 100644 --- a/git.c +++ b/git.c @@ -660,6 +660,7 @@ static struct cmd_struct commands[] = { { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, + { "survey", cmd_survey, RUN_SETUP }, { "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, { "tag", cmd_tag, RUN_SETUP | DELAY_PAGER_CONFIG }, diff --git a/git.rc.in b/git.rc.in index e69444eef3f0c5..cd671dc2f0d134 100644 --- a/git.rc.in +++ b/git.rc.in @@ -1,3 +1,4 @@ +#include 1 VERSIONINFO FILEVERSION @GIT_MAJOR_VERSION@,@GIT_MINOR_VERSION@,@GIT_MICRO_VERSION@,@GIT_PATCH_LEVEL@ PRODUCTVERSION @GIT_MAJOR_VERSION@,@GIT_MINOR_VERSION@,@GIT_MICRO_VERSION@,@GIT_PATCH_LEVEL@ @@ -12,6 +13,7 @@ BEGIN VALUE "OriginalFilename", "git.exe\0" VALUE "ProductName", "Git\0" VALUE "ProductVersion", "@GIT_VERSION@\0" + VALUE "FileVersion", "@GIT_VERSION@\0" END END diff --git a/grep.c b/grep.c index a54e5d86a96cfd..598c9cba61caa4 100644 --- a/grep.c +++ b/grep.c @@ -1647,6 +1647,8 @@ static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int colle bol = gs->buf; left = gs->size; + if (left && gs->buf[left-1] == '\n') + left--; while (left) { const char *eol; int hit; diff --git a/http.c b/http.c index ea9b16861bc3d4..d90565ba2469ff 100644 --- a/http.c +++ b/http.c @@ -131,7 +131,8 @@ enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL; static struct credential cert_auth = CREDENTIAL_INIT; static int ssl_cert_password_required; -static unsigned long http_auth_methods = CURLAUTH_ANY; +static unsigned long http_auth_any = CURLAUTH_ANY & ~CURLAUTH_NTLM; +static unsigned long http_auth_methods; static int http_auth_methods_restricted; /* Modes for which empty_auth cannot actually help us. */ static unsigned long empty_auth_useless = @@ -151,7 +152,12 @@ static char *cached_accept_language; static char *http_ssl_backend; -static int http_schannel_check_revoke = 1; +static long http_schannel_check_revoke_mode = +#ifdef CURLSSLOPT_REVOKE_BEST_EFFORT + CURLSSLOPT_REVOKE_BEST_EFFORT; +#else + CURLSSLOPT_NO_REVOKE; +#endif static long http_retry_after = 0; static long http_max_retries = 0; @@ -164,6 +170,8 @@ static long http_max_retry_time = 300; */ static int http_schannel_use_ssl_cainfo; +static int http_auto_client_cert; + static int always_auth_proactively(void) { return http_proactive_auth != PROACTIVE_AUTH_NONE && @@ -430,8 +438,29 @@ static int http_options(const char *var, const char *value, return 0; } + if (!strcmp("http.allowntlmauth", var)) { + if (git_config_bool(var, value)) { + http_auth_any |= CURLAUTH_NTLM; + } else { + http_auth_any &= ~CURLAUTH_NTLM; + } + return 0; + } + if (!strcmp("http.schannelcheckrevoke", var)) { - http_schannel_check_revoke = git_config_bool(var, value); + if (value && !strcmp(value, "best-effort")) { + http_schannel_check_revoke_mode = +#ifdef CURLSSLOPT_REVOKE_BEST_EFFORT + CURLSSLOPT_REVOKE_BEST_EFFORT; +#else + CURLSSLOPT_NO_REVOKE; + warning(_("%s=%s unsupported by current cURL"), + var, value); +#endif + } else + http_schannel_check_revoke_mode = + (git_config_bool(var, value) ? + 0 : CURLSSLOPT_NO_REVOKE); return 0; } @@ -440,6 +469,11 @@ static int http_options(const char *var, const char *value, return 0; } + if (!strcmp("http.sslautoclientcert", var)) { + http_auto_client_cert = git_config_bool(var, value); + return 0; + } + if (!strcmp("http.minsessions", var)) { min_curl_sessions = git_config_int(var, value, ctx->kvi); if (min_curl_sessions > 1) @@ -651,6 +685,11 @@ static void init_curl_http_auth(CURL *result) credential_fill(the_repository, &http_auth, 1); + if (http_auth.ntlm_allow && !(http_auth_methods & CURLAUTH_NTLM)) { + http_auth_methods |= CURLAUTH_NTLM; + curl_easy_setopt(result, CURLOPT_HTTPAUTH, http_auth_methods); + } + if (http_auth.password) { if (always_auth_proactively()) { /* @@ -726,11 +765,11 @@ static void init_curl_proxy_auth(CURL *result) if (i == ARRAY_SIZE(proxy_authmethods)) { warning("unsupported proxy authentication method %s: using anyauth", http_proxy_authmethod); - curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_PROXYAUTH, http_auth_any); } } else - curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_PROXYAUTH, http_auth_any); } static int has_cert_password(void) @@ -1140,7 +1179,7 @@ static CURL *get_curl_handle(void) } curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); - curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_HTTPAUTH, http_auth_any); #ifdef CURLGSSAPI_DELEGATION_FLAG if (curl_deleg) { @@ -1158,9 +1197,20 @@ static CURL *get_curl_handle(void) } #endif - if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) && - !http_schannel_check_revoke) { - curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_NO_REVOKE); + if (http_ssl_backend && !strcmp("schannel", http_ssl_backend)) { + long ssl_options = 0; + if (http_schannel_check_revoke_mode) { + ssl_options |= http_schannel_check_revoke_mode; + } + + if (http_auto_client_cert) { +#ifdef GIT_CURL_HAVE_CURLSSLOPT_AUTO_CLIENT_CERT + ssl_options |= CURLSSLOPT_AUTO_CLIENT_CERT; +#endif + } + + if (ssl_options) + curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, ssl_options); } if (http_proactive_auth != PROACTIVE_AUTH_NONE) @@ -1508,6 +1558,8 @@ void http_init(struct remote *remote, const char *url, int proactive_auth) set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES"); set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME"); + http_auth_methods = http_auth_any; + curl_default = get_curl_handle(); } @@ -1939,6 +1991,8 @@ static int handle_curl_result(struct slot_results *results) } else if (missing_target(results)) return HTTP_MISSING_TARGET; else if (results->http_code == 401) { + http_auth.ntlm_suppressed = (results->auth_avail & CURLAUTH_NTLM) && + !(http_auth_any & CURLAUTH_NTLM); if ((http_auth.username && http_auth.password) ||\ (http_auth.authtype && http_auth.credential)) { if (http_auth.multistage) { @@ -1948,6 +2002,16 @@ static int handle_curl_result(struct slot_results *results) credential_reject(the_repository, &http_auth); if (always_auth_proactively()) http_proactive_auth = PROACTIVE_AUTH_NONE; + if (http_auth.ntlm_suppressed) { + warning(_("Due to its cryptographic weaknesses, " + "NTLM authentication has been\n" + "disabled in Git by default. You can " + "re-enable it for trusted servers\n" + "by running:\n\n" + "git config set " + "http.%s://%s.allowNTLMAuth true"), + http_auth.protocol, http_auth.host); + } return HTTP_NOAUTH; } else { if (curl_empty_auth == -1 && @@ -2472,6 +2536,13 @@ static int http_request_recoverable(const char *url, http_reauth_prepare(1); } + /* + * Re-enable NTLM auth if the helper allows it and we would + * otherwise suppress authentication via NTLM. + */ + if (http_auth.ntlm_suppressed && http_auth.ntlm_allow) + http_auth_methods |= CURLAUTH_NTLM; + ret = http_request(url, result, target, options); } if (ret == HTTP_RATE_LIMITED) { diff --git a/mem-pool.c b/mem-pool.c index 8bc77cb0e80a35..89bca70f713692 100644 --- a/mem-pool.c +++ b/mem-pool.c @@ -7,7 +7,9 @@ #include "git-compat-util.h" #include "mem-pool.h" #include "gettext.h" +#include "trace.h" +static struct trace_key trace_mem_pool = TRACE_KEY_INIT(MEMPOOL); #define BLOCK_GROWTH_SIZE (1024 * 1024 - sizeof(struct mp_block)) /* @@ -65,12 +67,20 @@ void mem_pool_init(struct mem_pool *pool, size_t initial_size) if (initial_size > 0) mem_pool_alloc_block(pool, initial_size, NULL); + + trace_printf_key(&trace_mem_pool, + "mem_pool (%p): init (%"PRIuMAX") initial size\n", + (void *)pool, (uintmax_t)initial_size); } void mem_pool_discard(struct mem_pool *pool, int invalidate_memory) { struct mp_block *block, *block_to_free; + trace_printf_key(&trace_mem_pool, + "mem_pool (%p): discard (%"PRIuMAX") unused\n", + (void *)pool, + (uintmax_t)(pool->mp_block->end - pool->mp_block->next_free)); block = pool->mp_block; while (block) { diff --git a/meson.build b/meson.build index 064fe2e2f1f4e5..8d9be34e36563e 100644 --- a/meson.build +++ b/meson.build @@ -691,6 +691,7 @@ builtin_sources = [ 'builtin/stash.c', 'builtin/stripspace.c', 'builtin/submodule--helper.c', + 'builtin/survey.c', 'builtin/symbolic-ref.c', 'builtin/tag.c', 'builtin/unpack-file.c', @@ -1289,15 +1290,16 @@ elif host_machine.system() == 'windows' 'compat/winansi.c', 'compat/win32/dirent.c', 'compat/win32/flush.c', + 'compat/win32/fscache.c', 'compat/win32/path-utils.c', 'compat/win32/pthread.c', 'compat/win32/syslog.c', + 'compat/win32/wsl.c', 'compat/win32mmap.c', ] libgit_c_args += [ '-DDETECT_MSYS_TTY', - '-DENSURE_MSYSTEM_IS_SET', '-DNATIVE_CRLF', '-DNOGDI', '-DNO_POSIX_GOODIES', @@ -1307,6 +1309,18 @@ elif host_machine.system() == 'windows' '-D__USE_MINGW_ANSI_STDIO=0', ] + msystem = get_option('msystem') + if msystem != '' + mingw_prefix = get_option('mingw_prefix') + if mingw_prefix == '' + mingw_prefix = '/' + msystem.to_lower() + endif + libgit_c_args += [ + '-DENSURE_MSYSTEM_IS_SET="' + msystem + '"', + '-DMINGW_PREFIX="' + mingw_prefix + '"' + ] + endif + libgit_dependencies += compiler.find_library('ntdll') libgit_include_directories += 'compat/win32' if compiler.get_id() == 'msvc' diff --git a/meson_options.txt b/meson_options.txt index 80a8025f20be6e..642e7f5eaef761 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -21,6 +21,10 @@ option('runtime_prefix', type: 'boolean', value: false, description: 'Resolve ancillary tooling and support files relative to the location of the runtime binary instead of hard-coding them into the binary.') option('sane_tool_path', type: 'array', value: [], description: 'An array of paths to pick up tools from in case the normal tools are broken or lacking.') +option('msystem', type: 'string', value: '', + description: 'Fall-back on Windows when MSYSTEM is not set.') +option('mingw_prefix', type: 'string', value: '', + description: 'Fall-back on Windows when MINGW_PREFIX is not set.') # Build information compiled into Git and other parts like documentation. option('build_date', type: 'string', value: '', diff --git a/object-file.c b/object-file.c index 90f995d0000bf6..c648cecd80cdf4 100644 --- a/object-file.c +++ b/object-file.c @@ -561,9 +561,9 @@ int odb_source_loose_read_object_info(struct odb_source *source, } static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c, - const void *buf, unsigned long len, + const void *buf, size_t len, struct object_id *oid, - char *hdr, int *hdrlen) + char *hdr, size_t *hdrlen) { algo->init_fn(c); git_hash_update(c, hdr, *hdrlen); @@ -572,16 +572,16 @@ static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_c } static void write_object_file_prepare(const struct git_hash_algo *algo, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, struct object_id *oid, - char *hdr, int *hdrlen) + char *hdr, size_t *hdrlen) { struct git_hash_ctx c; /* Generate the header */ *hdrlen = format_object_header(hdr, *hdrlen, type, len); - /* Sha1.. */ + /* Hash (function pointers) computation */ hash_object_body(algo, &c, buf, len, oid, hdr, hdrlen); } @@ -717,11 +717,11 @@ int finalize_object_file_flags(struct repository *repo, } void hash_object_file(const struct git_hash_algo *algo, const void *buf, - unsigned long len, enum object_type type, + size_t len, enum object_type type, struct object_id *oid) { char hdr[MAX_HEADER_LEN]; - int hdrlen = sizeof(hdr); + size_t hdrlen = sizeof(hdr); write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen); } @@ -1177,7 +1177,7 @@ int odb_source_loose_write_stream(struct odb_source *source, } int odb_source_loose_write_object(struct odb_source *source, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, struct object_id *oid, struct object_id *compat_oid_in, enum odb_write_object_flags flags) @@ -1186,7 +1186,7 @@ int odb_source_loose_write_object(struct odb_source *source, const struct git_hash_algo *compat = source->odb->repo->compat_hash_algo; struct object_id compat_oid; char hdr[MAX_HEADER_LEN]; - int hdrlen = sizeof(hdr); + size_t hdrlen = sizeof(hdr); /* Generate compat_oid */ if (compat) { diff --git a/object-file.h b/object-file.h index 5241b8dd5c564d..e1e22d512d7e10 100644 --- a/object-file.h +++ b/object-file.h @@ -66,7 +66,7 @@ int odb_source_loose_freshen_object(struct odb_source *source, const struct object_id *oid); int odb_source_loose_write_object(struct odb_source *source, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, struct object_id *oid, struct object_id *compat_oid_in, enum odb_write_object_flags flags); @@ -201,7 +201,7 @@ int finalize_object_file_flags(struct repository *repo, enum finalize_object_file_flags flags); void hash_object_file(const struct git_hash_algo *algo, const void *buf, - unsigned long len, enum object_type type, + size_t len, enum object_type type, struct object_id *oid); /* Helper to check and "touch" a file */ diff --git a/parallel-checkout.c b/parallel-checkout.c index 0bf4bd6d4abd8c..8fadb7c804bc02 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -640,6 +640,7 @@ static void write_items_sequentially(struct checkout *state) { size_t i; + flush_fscache(); for (i = 0; i < parallel_checkout.nr; i++) { struct parallel_checkout_item *pc_item = ¶llel_checkout.items[i]; write_pc_item(pc_item, state); diff --git a/path.c b/path.c index d7e17bf17404de..f45263210986cb 100644 --- a/path.c +++ b/path.c @@ -1328,6 +1328,45 @@ char *strip_path_suffix(const char *path, const char *suffix) return offset == -1 ? NULL : xstrndup(path, offset); } +int is_mount_point_via_stat(struct strbuf *path) +{ + size_t len = path->len; + dev_t current_dev; + struct stat st; + + if (!strcmp("/", path->buf)) + return 1; + + strbuf_addstr(path, "/."); + if (lstat(path->buf, &st)) { + /* + * If we cannot access the current directory, we cannot say + * that it is a bind mount. + */ + strbuf_setlen(path, len); + return 0; + } + current_dev = st.st_dev; + + /* Now look at the parent directory */ + strbuf_addch(path, '.'); + if (lstat(path->buf, &st)) { + /* + * If we cannot access the parent directory, we cannot say + * that it is a bind mount. + */ + strbuf_setlen(path, len); + return 0; + } + strbuf_setlen(path, len); + + /* + * If the device ID differs between current and parent directory, + * then it is a bind mount. + */ + return current_dev != st.st_dev; +} + int daemon_avoid_alias(const char *p) { int sl, ndot; @@ -1545,6 +1584,7 @@ int looks_like_command_line_option(const char *str) char *xdg_config_home_for(const char *subdir, const char *filename) { const char *home, *config_home; + char *home_config = NULL; assert(subdir); assert(filename); @@ -1553,10 +1593,26 @@ char *xdg_config_home_for(const char *subdir, const char *filename) return mkpathdup("%s/%s/%s", config_home, subdir, filename); home = getenv("HOME"); - if (home) - return mkpathdup("%s/.config/%s/%s", home, subdir, filename); + if (home && *home) + home_config = mkpathdup("%s/.config/%s/%s", home, subdir, filename); + + #ifdef WIN32 + { + const char *appdata = getenv("APPDATA"); + if (appdata && *appdata) { + char *appdata_config = mkpathdup("%s/Git/%s", appdata, filename); + if (file_exists(appdata_config)) { + if (home_config && file_exists(home_config)) + warning("'%s' was ignored because '%s' exists.", home_config, appdata_config); + free(home_config); + return appdata_config; + } + free(appdata_config); + } + } + #endif - return NULL; + return home_config; } char *xdg_config_home(const char *filename) diff --git a/path.h b/path.h index 0434ba5e07e806..85713809f63624 100644 --- a/path.h +++ b/path.h @@ -161,6 +161,7 @@ int normalize_path_copy(char *dst, const char *src); int strbuf_normalize_path(struct strbuf *src); int longest_ancestor_length(const char *path, struct string_list *prefixes); char *strip_path_suffix(const char *path, const char *suffix); +int is_mount_point_via_stat(struct strbuf *path); int daemon_avoid_alias(const char *path); /* diff --git a/preload-index.c b/preload-index.c index b222821b448526..ac0310008754a3 100644 --- a/preload-index.c +++ b/preload-index.c @@ -20,6 +20,8 @@ #include "trace2.h" #include "config.h" +static struct fscache *fscache; + /* * Mostly randomly chosen maximum thread counts: we * cap the parallelism to 20 threads, and we want @@ -57,6 +59,7 @@ static void *preload_thread(void *_data) nr = index->cache_nr - p->offset; last_nr = nr; + enable_fscache(nr); do { struct cache_entry *ce = *cep++; struct stat st; @@ -100,6 +103,7 @@ static void *preload_thread(void *_data) pthread_mutex_unlock(&pd->mutex); } cache_def_clear(&cache); + merge_fscache(fscache); return NULL; } @@ -118,6 +122,7 @@ void preload_index(struct index_state *index, if (!HAVE_THREADS || !core_preload_index) return; + fscache = getcache_fscache(); threads = index->cache_nr / THREAD_COST; if ((index->cache_nr > 1) && (threads < 2) && git_env_bool("GIT_TEST_PRELOAD_INDEX", 0)) threads = 2; diff --git a/read-cache.c b/read-cache.c index 21829102ae275e..ea5f839156b53e 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1516,6 +1516,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, typechange_fmt = in_porcelain ? "T\t%s\n" : "%s: needs update\n"; added_fmt = in_porcelain ? "A\t%s\n" : "%s: needs update\n"; unmerged_fmt = in_porcelain ? "U\t%s\n" : "%s: needs merge\n"; + enable_fscache(0); /* * Use the multi-threaded preload_index() to refresh most of the * cache entries quickly then in the single threaded loop below, @@ -1610,6 +1611,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, display_progress(progress, istate->cache_nr); stop_progress(&progress); trace_performance_leave("refresh index"); + disable_fscache(); return has_errors; } diff --git a/refs/files-backend.c b/refs/files-backend.c index a4c7858787127d..be15e3ad413632 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2099,7 +2099,7 @@ static int create_ref_symlink(struct ref_lock *lock, const char *target) ref_path = get_locked_file_path(&lock->lk); unlink(ref_path); - ret = symlink(target, ref_path); + ret = create_symlink(NULL, target, ref_path); free(ref_path); if (ret) diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index 4ae22922de558b..c245b16fbd831b 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -380,6 +380,8 @@ static struct ref_store *reftable_be_init(struct repository *repo, mask = umask(0); umask(mask); + reftable_set_alloc(malloc, realloc, free); + refs_compute_filesystem_location(gitdir, payload, &is_worktree, &refdir, &ref_common_dir); diff --git a/repository.c b/repository.c index db57b8308b94e7..47debec1c1bcc7 100644 --- a/repository.c +++ b/repository.c @@ -153,7 +153,7 @@ static void repo_set_commondir(struct repository *repo, { struct strbuf sb = STRBUF_INIT; - free(repo->commondir); + FREE_AND_NULL(repo->commondir); if (commondir) { repo->different_commondir = 1; diff --git a/run-command.c b/run-command.c index e70a8a387b9042..ceb33119655de9 100644 --- a/run-command.c +++ b/run-command.c @@ -582,6 +582,7 @@ static int wait_or_whine(pid_t pid, const char *argv0, int in_signal) */ code += 128; } else if (WIFEXITED(status)) { + warn_about_git_lfs_on_windows7(status, argv0); code = WEXITSTATUS(status); } else { if (!in_signal) diff --git a/send-pack.c b/send-pack.c index 3bb5afc687aaf0..b7f6a7dc00a663 100644 --- a/send-pack.c +++ b/send-pack.c @@ -522,7 +522,7 @@ int send_pack(struct repository *r, int need_pack_data = 0; int allow_deleting_refs = 0; int status_report = 0; - int use_sideband = 0; + int use_sideband = 1; int quiet_supported = 0; int agent_supported = 0; int advertise_sid = 0; @@ -546,6 +546,7 @@ int send_pack(struct repository *r, goto out; } + repo_config_get_bool(r, "sendpack.sideband", &use_sideband); repo_config_get_bool(r, "push.negotiate", &push_negotiate); if (push_negotiate) { trace2_region_enter("send_pack", "push_negotiate", r); @@ -570,8 +571,7 @@ int send_pack(struct repository *r, allow_deleting_refs = 1; if (server_supports("ofs-delta")) args->use_ofs_delta = 1; - if (server_supports("side-band-64k")) - use_sideband = 1; + use_sideband = use_sideband && server_supports("side-band-64k"); if (server_supports("quiet")) quiet_supported = 1; if (server_supports("agent")) diff --git a/setup.c b/setup.c index 075bf89fa99af9..25e417740ffa53 100644 --- a/setup.c +++ b/setup.c @@ -1956,10 +1956,19 @@ const char *setup_git_directory_gently(struct repository *repo, int *nongit_ok) break; case GIT_DIR_INVALID_OWNERSHIP: if (!nongit_ok) { + struct strbuf prequoted = STRBUF_INIT; struct strbuf quoted = STRBUF_INIT; strbuf_complete(&report, '\n'); - sq_quote_buf_pretty("ed, dir.buf); + +#ifdef __MINGW32__ + if (dir.buf[0] == '/') + strbuf_addstr(&prequoted, "%(prefix)/"); +#endif + + strbuf_add(&prequoted, dir.buf, dir.len); + sq_quote_buf_pretty("ed, prequoted.buf); + die(_("detected dubious ownership in repository at '%s'\n" "%s" "To add an exception for this directory, call:\n" @@ -2321,7 +2330,7 @@ static void copy_templates_1(struct repository *repo, if (strbuf_readlink(&lnk, template_path->buf, st_template.st_size) < 0) die_errno(_("cannot readlink '%s'"), template_path->buf); - if (symlink(lnk.buf, path->buf)) + if (create_symlink(NULL, lnk.buf, path->buf)) die_errno(_("cannot symlink '%s' '%s'"), lnk.buf, path->buf); strbuf_release(&lnk); @@ -2603,7 +2612,7 @@ static int create_default_files(struct repository *repo, repo_git_path_replace(repo, &path, "tXXXXXX"); if (!close(xmkstemp(path.buf)) && !unlink(path.buf) && - !symlink("testing", path.buf) && + !create_symlink(NULL, "testing", path.buf) && !lstat(path.buf, &st1) && S_ISLNK(st1.st_mode)) unlink(path.buf); /* good */ diff --git a/sha1dc_git.c b/sha1dc_git.c index 9b675a046ee699..fe58d7962a30c9 100644 --- a/sha1dc_git.c +++ b/sha1dc_git.c @@ -27,10 +27,9 @@ void git_SHA1DCFinal(unsigned char hash[20], SHA1_CTX *ctx) /* * Same as SHA1DCUpdate, but adjust types to match git's usual interface. */ -void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *vdata, unsigned long len) +void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *vdata, size_t len) { const char *data = vdata; - /* We expect an unsigned long, but sha1dc only takes an int */ while (len > INT_MAX) { SHA1DCUpdate(ctx, data, INT_MAX); data += INT_MAX; diff --git a/sha1dc_git.h b/sha1dc_git.h index f6f880cabea382..0bcf1aa84b7241 100644 --- a/sha1dc_git.h +++ b/sha1dc_git.h @@ -15,7 +15,7 @@ void git_SHA1DCInit(SHA1_CTX *); #endif void git_SHA1DCFinal(unsigned char [20], SHA1_CTX *); -void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *data, unsigned long len); +void git_SHA1DCUpdate(SHA1_CTX *ctx, const void *data, size_t len); #define platform_SHA_IS_SHA1DC /* used by "test-tool sha1-is-sha1dc" */ diff --git a/t/README b/t/README index adbbd9acf4ab27..f19468151410eb 100644 --- a/t/README +++ b/t/README @@ -479,6 +479,9 @@ GIT_TEST_NAME_HASH_VERSION=, when set, causes 'git pack-objects' to assume '--name-hash-version='. +GIT_TEST_FSCACHE= exercises the uncommon fscache code path +which adds a cache below mingw's lstat and dirent implementations. + Naming Tests ------------ diff --git a/t/helper/meson.build b/t/helper/meson.build index 3235f10ab8aae1..d4499d26a9af1f 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -29,6 +29,7 @@ test_tool_sources = [ 'test-hash.c', 'test-hashmap.c', 'test-hexdump.c', + 'test-iconv.c', 'test-json-writer.c', 'test-lazy-init-name-hash.c', 'test-match-trees.c', diff --git a/t/helper/test-iconv.c b/t/helper/test-iconv.c new file mode 100644 index 00000000000000..d3c772fddf990b --- /dev/null +++ b/t/helper/test-iconv.c @@ -0,0 +1,47 @@ +#include "test-tool.h" +#include "git-compat-util.h" +#include "strbuf.h" +#include "gettext.h" +#include "parse-options.h" +#include "utf8.h" + +int cmd__iconv(int argc, const char **argv) +{ + struct strbuf buf = STRBUF_INIT; + char *from = NULL, *to = NULL, *p; + size_t len; + int ret = 0; + const char * const iconv_usage[] = { + N_("test-helper --iconv []"), + NULL + }; + struct option options[] = { + OPT_STRING('f', "from-code", &from, "encoding", "from"), + OPT_STRING('t', "to-code", &to, "encoding", "to"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, + iconv_usage, 0); + + if (argc > 1 || !from || !to) + usage_with_options(iconv_usage, options); + + if (!argc) { + if (strbuf_read(&buf, 0, 2048) < 0) + die_errno("Could not read from stdin"); + } else if (strbuf_read_file(&buf, argv[0], 2048) < 0) + die_errno("Could not read from '%s'", argv[0]); + + p = reencode_string_len(buf.buf, buf.len, to, from, &len); + if (!p) + die_errno("Could not reencode"); + if (write(1, p, len) < 0) + ret = !!error_errno("Could not write %"PRIuMAX" bytes", + (uintmax_t)len); + + strbuf_release(&buf); + free(p); + + return ret; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index b71a22b43bbc9e..ee16b2cb23719e 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -39,6 +39,7 @@ static struct test_cmd cmds[] = { { "hashmap", cmd__hashmap }, { "hash-speed", cmd__hash_speed }, { "hexdump", cmd__hexdump }, + { "iconv", cmd__iconv }, { "json-writer", cmd__json_writer }, { "lazy-init-name-hash", cmd__lazy_init_name_hash }, { "match-trees", cmd__match_trees }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index f2885b33d58aa8..4cf9f935a4cdfa 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -32,6 +32,7 @@ int cmd__getcwd(int argc, const char **argv); int cmd__hashmap(int argc, const char **argv); int cmd__hash_speed(int argc, const char **argv); int cmd__hexdump(int argc, const char **argv); +int cmd__iconv(int argc, const char **argv); int cmd__json_writer(int argc, const char **argv); int cmd__lazy_init_name_hash(int argc, const char **argv); int cmd__match_trees(int argc, const char **argv); diff --git a/t/interop/interop-lib.sh b/t/interop/interop-lib.sh index 1b5864d2a7f22c..1facc69d97741a 100644 --- a/t/interop/interop-lib.sh +++ b/t/interop/interop-lib.sh @@ -4,6 +4,10 @@ . ../../GIT-BUILD-OPTIONS INTEROP_ROOT=$(pwd) BUILD_ROOT=$INTEROP_ROOT/build +case "$PATH" in +*\;*) PATH_SEP=\; ;; +*) PATH_SEP=: ;; +esac build_version () { if test -z "$1" @@ -57,7 +61,7 @@ wrap_git () { write_script "$1" <<-EOF GIT_EXEC_PATH="$2" export GIT_EXEC_PATH - PATH="$2:\$PATH" + PATH="$2$PATH_SEP\$PATH" export GIT_EXEC_PATH exec git "\$@" EOF @@ -71,7 +75,7 @@ generate_wrappers () { echo >&2 fatal: test tried to run generic git: $* exit 1 EOF - PATH=$(pwd)/.bin:$PATH + PATH=$(pwd)/.bin$PATH_SEP$PATH } VERSION_A=${GIT_TEST_VERSION_A:-$VERSION_A} diff --git a/t/test-binary-1.png b/t/lib-diff/test-binary-1.png similarity index 100% rename from t/test-binary-1.png rename to t/lib-diff/test-binary-1.png diff --git a/t/test-binary-2.png b/t/lib-diff/test-binary-2.png similarity index 100% rename from t/test-binary-2.png rename to t/lib-diff/test-binary-2.png diff --git a/t/lib-httpd.sh b/t/lib-httpd.sh index fc646447d5c038..68823c6ed2e200 100644 --- a/t/lib-httpd.sh +++ b/t/lib-httpd.sh @@ -168,6 +168,7 @@ prepare_httpd() { install_script apply-one-time-script.sh install_script nph-custom-auth.sh install_script http-429.sh + install_script ntlm-handshake.sh ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules" diff --git a/t/lib-httpd/apache.conf b/t/lib-httpd/apache.conf index 40a690b0bb7c9b..7a5c3620cfe901 100644 --- a/t/lib-httpd/apache.conf +++ b/t/lib-httpd/apache.conf @@ -155,6 +155,13 @@ SetEnv PERL_PATH ${PERL_PATH} CGIPassAuth on + + SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH} + SetEnv GIT_HTTP_EXPORT_ALL + + CGIPassAuth on + + ScriptAlias /smart/incomplete_length/git-upload-pack incomplete-length-upload-pack-v2-http.sh/ ScriptAlias /smart/incomplete_body/git-upload-pack incomplete-body-upload-pack-v2-http.sh/ ScriptAlias /smart/no_report/git-receive-pack error-no-report.sh/ @@ -166,6 +173,7 @@ ScriptAlias /error/ error.sh/ ScriptAliasMatch /one_time_script/(.*) apply-one-time-script.sh/$1 ScriptAliasMatch /http_429/(.*) http-429.sh/$1 ScriptAliasMatch /custom_auth/(.*) nph-custom-auth.sh/$1 +ScriptAliasMatch /ntlm_auth/(.*) ntlm-handshake.sh/$1 Options FollowSymlinks diff --git a/t/lib-httpd/ntlm-handshake.sh b/t/lib-httpd/ntlm-handshake.sh new file mode 100755 index 00000000000000..3cf1266e40f20a --- /dev/null +++ b/t/lib-httpd/ntlm-handshake.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +case "$HTTP_AUTHORIZATION" in +'') + # No Authorization header -> send NTLM challenge + echo "Status: 401 Unauthorized" + echo "WWW-Authenticate: NTLM" + echo + ;; +"NTLM TlRMTVNTUAAB"*) + # Type 1 -> respond with Type 2 challenge (hardcoded) + echo "Status: 401 Unauthorized" + # Base64-encoded version of the Type 2 challenge: + # signature: 'NTLMSSP\0' + # message_type: 2 + # target_name: 'NTLM-GIT-SERVER' + # flags: 0xa2898205 = + # NEGOTIATE_UNICODE, REQUEST_TARGET, NEGOTIATE_NT_ONLY, + # TARGET_TYPE_SERVER, TARGET_TYPE_SHARE, REQUEST_NON_NT_SESSION_KEY, + # NEGOTIATE_VERSION, NEGOTIATE_128, NEGOTIATE_56 + # challenge: 0xfa3dec518896295b + # context: '0000000000000000' + # target_info_present: true + # target_info_len: 128 + # version: '10.0 (build 19041)' + echo "WWW-Authenticate: NTLM TlRMTVNTUAACAAAAHgAeADgAAAAFgomi+j3sUYiWKVsAAAAAAAAAAIAAgABWAAAACgBhSgAAAA9OAFQATABNAC0ARwBJAFQALQBTAEUAUgBWAEUAUgACABIAVwBPAFIASwBHAFIATwBVAFAAAQAeAE4AVABMAE0ALQBHAEkAVAAtAFMARQBSAFYARQBSAAQAEgBXAE8AUgBLAEcAUgBPAFUAUAADAB4ATgBUAEwATQAtAEcASQBUAC0AUwBFAFIAVgBFAFIABwAIAACfOcZKYNwBAAAAAA==" + echo + ;; +"NTLM TlRMTVNTUAAD"*) + # Type 3 -> accept without validation + exec "$GIT_EXEC_PATH"/git-http-backend + ;; +*) + echo "Status: 500 Unrecognized" + echo + echo "Unhandled auth: '$HTTP_AUTHORIZATION'" + ;; +esac diff --git a/t/lib-proto-disable.sh b/t/lib-proto-disable.sh index 890622be81642b..9db481e1be15b2 100644 --- a/t/lib-proto-disable.sh +++ b/t/lib-proto-disable.sh @@ -214,7 +214,7 @@ setup_ext_wrapper () { cd "$TRASH_DIRECTORY/remote" && eval "$*" EOF - PATH=$TRASH_DIRECTORY:$PATH && + PATH=$TRASH_DIRECTORY$PATH_SEP$PATH && export TRASH_DIRECTORY ' } diff --git a/t/meson.build b/t/meson.build index 2af8d0127991db..bd4b2e9c696131 100644 --- a/t/meson.build +++ b/t/meson.build @@ -6,6 +6,7 @@ clar_test_suites = [ 'unit-tests/u-hashmap.c', 'unit-tests/u-list-objects-filter-options.c', 'unit-tests/u-mem-pool.c', + 'unit-tests/u-mingw.c', 'unit-tests/u-odb-inmemory.c', 'unit-tests/u-oid-array.c', 'unit-tests/u-oidmap.c', @@ -275,6 +276,8 @@ integration_tests = [ 't2026-checkout-pathspec-file.sh', 't2027-checkout-track.sh', 't2030-unresolve-info.sh', + 't2031-checkout-long-paths.sh', + 't2040-checkout-symlink-attr.sh', 't2050-git-dir-relative.sh', 't2060-switch.sh', 't2070-restore.sh', @@ -875,6 +878,7 @@ integration_tests = [ 't7105-reset-patch.sh', 't7106-reset-unborn-branch.sh', 't7107-reset-pathspec-file.sh', + 't7108-reset-stdin.sh', 't7110-reset-merge.sh', 't7111-reset-table.sh', 't7112-reset-submodule.sh', @@ -905,6 +909,7 @@ integration_tests = [ 't7424-submodule-mixed-ref-formats.sh', 't7425-submodule-gitdir-path-extension.sh', 't7426-submodule-get-default-remote.sh', + 't7429-submodule-long-path.sh', 't7450-bad-git-dotfiles.sh', 't7500-commit-template-squash-signoff.sh', 't7501-commit-basic-functionality.sh', @@ -980,6 +985,7 @@ integration_tests = [ 't8014-blame-ignore-fuzzy.sh', 't8015-blame-diff-algorithm.sh', 't8020-last-modified.sh', + 't8100-git-survey.sh', 't9001-send-email.sh', 't9002-column.sh', 't9003-help-autocorrect.sh', diff --git a/t/t0014-alias.sh b/t/t0014-alias.sh index 5144b0effd78aa..02a196f5e3ce1c 100755 --- a/t/t0014-alias.sh +++ b/t/t0014-alias.sh @@ -52,10 +52,10 @@ test_expect_success 'looping aliases - deprecated builtins' ' #' test_expect_success 'run-command formats empty args properly' ' - test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && - sed -ne "/run_command:/s/.*trace: run_command: //p" actual.raw >actual && - echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && - test_cmp expect actual + test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && + sed -ne "/run_command: git-frotz/s/.*trace: run_command: //p" actual.raw >actual && + echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && + test_cmp expect actual ' test_expect_success 'tracing a shell alias with arguments shows trace of prepared command' ' diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index f0d50d769e9fc5..0c5975336f2104 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -8,7 +8,7 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh . "$TEST_DIRECTORY"/lib-terminal.sh -PATH=$PWD:$PATH +PATH=$PWD$PATH_SEP$PATH TEST_ROOT="$(pwd)" write_script <<\EOF "$TEST_ROOT/rot13.sh" diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 8545cdfab559b4..5abfa202c19dca 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -147,25 +147,25 @@ ancestor /foo /fo -1 ancestor /foo /foo -1 ancestor /foo /bar -1 ancestor /foo /foo/bar -1 -ancestor /foo /foo:/bar -1 -ancestor /foo /:/foo:/bar 0 -ancestor /foo /foo:/:/bar 0 -ancestor /foo /:/bar:/foo 0 +ancestor /foo "/foo$PATH_SEP/bar" -1 +ancestor /foo "/$PATH_SEP/foo$PATH_SEP/bar" 0 +ancestor /foo "/foo$PATH_SEP/$PATH_SEP/bar" 0 +ancestor /foo "/$PATH_SEP/bar$PATH_SEP/foo" 0 ancestor /foo/bar / 0 ancestor /foo/bar /fo -1 ancestor /foo/bar /foo 4 ancestor /foo/bar /foo/ba -1 -ancestor /foo/bar /:/fo 0 -ancestor /foo/bar /foo:/foo/ba 4 +ancestor /foo/bar "/$PATH_SEP/fo" 0 +ancestor /foo/bar "/foo$PATH_SEP/foo/ba" 4 ancestor /foo/bar /bar -1 ancestor /foo/bar /fo -1 -ancestor /foo/bar /foo:/bar 4 -ancestor /foo/bar /:/foo:/bar 4 -ancestor /foo/bar /foo:/:/bar 4 -ancestor /foo/bar /:/bar:/fo 0 -ancestor /foo/bar /:/bar 0 +ancestor /foo/bar "/foo$PATH_SEP/bar" 4 +ancestor /foo/bar "/$PATH_SEP/foo$PATH_SEP/bar" 4 +ancestor /foo/bar "/foo$PATH_SEP/$PATH_SEP/bar" 4 +ancestor /foo/bar "/$PATH_SEP/bar$PATH_SEP/fo" 0 +ancestor /foo/bar "/$PATH_SEP/bar" 0 ancestor /foo/bar /foo 4 -ancestor /foo/bar /foo:/bar 4 +ancestor /foo/bar "/foo$PATH_SEP/bar" 4 ancestor /foo/bar /bar -1 # Windows-specific: DOS drives, network shares @@ -281,6 +281,14 @@ test_expect_success SYMLINKS 'real path works on symlinks' ' test_cmp expect actual ' +test_expect_success MINGW 'real path works near drive root' ' + # we need a non-existing path at the drive root; simply skip if C:/xyz exists + if test ! -e C:/xyz + then + test C:/xyz = $(test-tool path-utils real_path C:/xyz) + fi +' + test_expect_success SYMLINKS 'prefix_path works with absolute paths to work tree symlinks' ' ln -s target symlink && echo "symlink" >expect && @@ -602,7 +610,8 @@ test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD 'RUNTIME_PREFIX wor echo "echo HERE" | write_script pretend/libexec/git-core/git-here && GIT_EXEC_PATH= ./pretend/bin/git here >actual && echo HERE >expect && - test_cmp expect actual' + test_cmp expect actual +' test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD '%(prefix)/ works' ' git config yes.path "%(prefix)/yes" && @@ -611,4 +620,34 @@ test_expect_success !VALGRIND,RUNTIME_PREFIX,CAN_EXEC_IN_PWD '%(prefix)/ works' test_cmp expect actual ' +test_expect_success MINGW,RUNTIME_PREFIX 'MSYSTEM/PATH is adjusted if necessary' ' + if test -z "$MINGW_PREFIX" + then + MINGW_PREFIX="/$(echo "${MSYSTEM:-MINGW64}" | tr A-Z a-z)" + fi && + mkdir -p "$HOME"/bin pretend"$MINGW_PREFIX"/bin \ + pretend"$MINGW_PREFIX"/libexec/git-core pretend/usr/bin && + cp "$GIT_EXEC_PATH"/git.exe pretend"$MINGW_PREFIX"/bin/ && + cp "$GIT_EXEC_PATH"/git.exe pretend"$MINGW_PREFIX"/libexec/git-core/ && + # copy the .dll files, if any (happens when building via CMake) + if test -n "$(ls "$GIT_EXEC_PATH"/*.dll 2>/dev/null)" + then + cp "$GIT_EXEC_PATH"/*.dll pretend"$MINGW_PREFIX"/bin/ && + cp "$GIT_EXEC_PATH"/*.dll pretend"$MINGW_PREFIX"/libexec/git-core/ + fi && + echo "env | grep MSYSTEM=" | write_script "$HOME"/bin/git-test-home && + echo "echo ${MINGW_PREFIX#/}" | write_script pretend"$MINGW_PREFIX"/bin/git-test-bin && + echo "echo usr" | write_script pretend/usr/bin/git-test-bin2 && + + ( + MSYSTEM= && + GIT_EXEC_PATH= && + pretend"$MINGW_PREFIX"/libexec/git-core/git.exe test-home >actual && + pretend"$MINGW_PREFIX"/libexec/git-core/git.exe test-bin >>actual && + pretend"$MINGW_PREFIX"/bin/git.exe test-bin2 >>actual + ) && + test_write_lines MSYSTEM=$MSYSTEM "${MINGW_PREFIX#/}" usr >expect && + test_cmp expect actual +' + test_done diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index 60cfe65979e215..905e90e1f72541 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -69,7 +69,7 @@ test_expect_success 'run_command does not try to execute a directory' ' cat bin2/greet EOF - PATH=$PWD/bin1:$PWD/bin2:$PATH \ + PATH=$PWD/bin1$PATH_SEP$PWD/bin2$PATH_SEP$PATH \ test-tool run-command run-command greet >actual 2>err && test_cmp bin2/greet actual && test_must_be_empty err @@ -86,7 +86,7 @@ test_expect_success POSIXPERM 'run_command passes over non-executable file' ' cat bin2/greet EOF - PATH=$PWD/bin1:$PWD/bin2:$PATH \ + PATH=$PWD/bin1$PATH_SEP$PWD/bin2$PATH_SEP$PATH \ test-tool run-command run-command greet >actual 2>err && test_cmp bin2/greet actual && test_must_be_empty err @@ -106,7 +106,7 @@ test_expect_success POSIXPERM,SANITY 'unreadable directory in PATH' ' git config alias.nitfol "!echo frotz" && chmod a-rx local-command && ( - PATH=./local-command:$PATH && + PATH=./local-command$PATH_SEP$PATH && git nitfol >actual ) && echo frotz >expect && diff --git a/t/t0300-credentials.sh b/t/t0300-credentials.sh index 64ead1571ae1e1..add4aeb6f92fd1 100755 --- a/t/t0300-credentials.sh +++ b/t/t0300-credentials.sh @@ -80,7 +80,7 @@ test_expect_success 'setup helper scripts' ' printf "username=\\007latrix Lestrange\\n" EOF - PATH="$PWD:$PATH" + PATH="$PWD$PATH_SEP$PATH" ' test_expect_success 'credential_fill invokes helper' ' diff --git a/t/t0301-credential-cache.sh b/t/t0301-credential-cache.sh index 6f7cfd9e33f633..a14032626192d0 100755 --- a/t/t0301-credential-cache.sh +++ b/t/t0301-credential-cache.sh @@ -12,7 +12,7 @@ test -z "$NO_UNIX_SOCKETS" || { if test_have_prereq MINGW then service_running=$(sc query afunix | grep "4 RUNNING") - test -z "$service_running" || { + test -n "$service_running" || { skip_all='skipping credential-cache tests, unix sockets not available' test_done } diff --git a/t/t1007-hash-object.sh b/t/t1007-hash-object.sh index de076293b62a76..841a6671d1a3c1 100755 --- a/t/t1007-hash-object.sh +++ b/t/t1007-hash-object.sh @@ -49,6 +49,9 @@ test_expect_success 'setup' ' example sha1:ddd3f836d3e3fbb7ae289aa9ae83536f76956399 example sha256:b44fe1fe65589848253737db859bd490453510719d7424daab03daf0767b85ae + + large5GB sha1:0be2be10a4c8764f32c4bf372a98edc731a4b204 + large5GB sha256:dc18ca621300c8d3cfa505a275641ebab00de189859e022a975056882d313e64 EOF ' @@ -258,4 +261,40 @@ test_expect_success '--stdin outside of repository (uses default hash)' ' test_cmp expect actual ' +test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ + 'files over 4GB hash literally' ' + test-tool genzeros $((5*1024*1024*1024)) >big && + test_oid large5GB >expect && + git hash-object --stdin --literally actual && + test_cmp expect actual +' + +test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ + 'files over 4GB hash correctly via --stdin' ' + { test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } && + test_oid large5GB >expect && + git hash-object --stdin actual && + test_cmp expect actual +' + +test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ + 'files over 4GB hash correctly' ' + { test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } && + test_oid large5GB >expect && + git hash-object -- big >actual && + test_cmp expect actual +' + +# This clean filter does nothing, other than excercising the interface. +# We ensure that cleaning doesn't mangle large files on 64-bit Windows. +test_expect_success EXPENSIVE,SIZE_T_IS_64BIT,!LONG_IS_64BIT \ + 'hash filtered files over 4GB correctly' ' + { test -f big || test-tool genzeros $((5*1024*1024*1024)) >big; } && + test_oid large5GB >expect && + test_config filter.null-filter.clean "cat" && + echo "big filter=null-filter" >.gitattributes && + git hash-object -- big >actual && + test_cmp expect actual +' + test_done diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 3a14218b245d4c..529844e2862c74 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -106,4 +106,24 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success MINGW 'no unnecessary opendir() with fscache' ' + git clone . fscache-test && + ( + cd fscache-test && + git config core.fscache 1 && + echo "/excluded/*" >.git/info/sparse-checkout && + for f in $(test_seq 10) + do + sha1=$(echo $f | git hash-object -w --stdin) && + git update-index --add \ + --cacheinfo 100644,$sha1,excluded/$f || exit 1 + done && + test_tick && + git commit -m excluded && + GIT_TRACE_FSCACHE=1 git status >out 2>err && + grep excluded err >grep.out && + test_line_count = 1 grep.out + ) +' + test_done diff --git a/t/t1504-ceiling-dirs.sh b/t/t1504-ceiling-dirs.sh index e04420f4368b93..ff9fb804827b59 100755 --- a/t/t1504-ceiling-dirs.sh +++ b/t/t1504-ceiling-dirs.sh @@ -84,9 +84,9 @@ then GIT_CEILING_DIRECTORIES="$TRASH_ROOT/top/" test_fail subdir_ceil_at_top_slash - GIT_CEILING_DIRECTORIES=":$TRASH_ROOT/top" + GIT_CEILING_DIRECTORIES="$PATH_SEP$TRASH_ROOT/top" test_prefix subdir_ceil_at_top_no_resolve "sub/dir/" - GIT_CEILING_DIRECTORIES=":$TRASH_ROOT/top/" + GIT_CEILING_DIRECTORIES="$PATH_SEP$TRASH_ROOT/top/" test_prefix subdir_ceil_at_top_slash_no_resolve "sub/dir/" fi @@ -116,13 +116,13 @@ GIT_CEILING_DIRECTORIES="$TRASH_ROOT/subdi" test_prefix subdir_ceil_at_subdi_slash "sub/dir/" -GIT_CEILING_DIRECTORIES="/foo:$TRASH_ROOT/sub" +GIT_CEILING_DIRECTORIES="/foo$PATH_SEP$TRASH_ROOT/sub" test_fail second_of_two -GIT_CEILING_DIRECTORIES="$TRASH_ROOT/sub:/bar" +GIT_CEILING_DIRECTORIES="$TRASH_ROOT/sub$PATH_SEP/bar" test_fail first_of_two -GIT_CEILING_DIRECTORIES="/foo:$TRASH_ROOT/sub:/bar" +GIT_CEILING_DIRECTORIES="/foo$PATH_SEP$TRASH_ROOT/sub$PATH_SEP/bar" test_fail second_of_three diff --git a/t/t1517-outside-repo.sh b/t/t1517-outside-repo.sh index c557f2f55c2175..b2715eb0192311 100755 --- a/t/t1517-outside-repo.sh +++ b/t/t1517-outside-repo.sh @@ -129,7 +129,7 @@ do merge-octopus | merge-one-file | merge-resolve | mergetool | \ mktag | p4 | p4.py | pickaxe | remote-ftp | remote-ftps | \ remote-http | remote-https | replay | send-email | \ - sh-i18n--envsubst | shell | show | stage | submodule | svn | \ + sh-i18n--envsubst | shell | show | stage | submodule | survey | svn | \ upload-archive--writer | upload-pack | web--browse | whatchanged) expect_outcome=expect_failure ;; *) diff --git a/t/t2031-checkout-long-paths.sh b/t/t2031-checkout-long-paths.sh new file mode 100755 index 00000000000000..15416a1d6ee8c7 --- /dev/null +++ b/t/t2031-checkout-long-paths.sh @@ -0,0 +1,111 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + grep -q "Filename too long" error && + test ! -d longpa* +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa*/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >>$(ls longpa*/longtestfile) && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + if test -z "$debug" + then + rm -rf longpa~1 + fi +' + +# check that the template used in the test won't be too long: +abspath="$(pwd)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_expect_success SYMLINKS_WINDOWS 'leave drive-less, short paths intact' ' + printf "/Program Files" >symlink-target && + symlink_target_oid="$(git hash-object -w --stdin actual && + grep " *PF *\\[\\\\Program Files\\]" actual +' + +test_done diff --git a/t/t2040-checkout-symlink-attr.sh b/t/t2040-checkout-symlink-attr.sh new file mode 100755 index 00000000000000..e00c31d096ce88 --- /dev/null +++ b/t/t2040-checkout-symlink-attr.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +test_description='checkout symlinks with `symlink` attribute on Windows + +Ensures that Git for Windows creates symlinks of the right type, +as specified by the `symlink` attribute in `.gitattributes`.' + +# Tell MSYS to create native symlinks. Without this flag test-lib's +# prerequisite detection for SYMLINKS doesn't detect the right thing. +MSYS=winsymlinks:nativestrict && export MSYS + +. ./test-lib.sh + +if ! test_have_prereq MINGW,SYMLINKS +then + skip_all='skipping $0: MinGW-only test, which requires symlink support.' + test_done +fi + +# Adds a symlink to the index without clobbering the work tree. +cache_symlink () { + sha=$(printf '%s' "$1" | git hash-object --stdin -w) && + git update-index --add --cacheinfo 120000,$sha,"$2" +} + +test_expect_success 'checkout symlinks with attr' ' + cache_symlink file1 file-link && + cache_symlink dir dir-link && + + printf "file-link symlink=file\ndir-link symlink=dir\n" >.gitattributes && + git add .gitattributes && + + git checkout . && + + mkdir dir && + echo "[a]b=c" >file1 && + echo "[x]y=z" >dir/file2 && + + # MSYS2 is very forgiving, it will resolve symlinks even if the + # symlink type is incorrect. To make this test meaningful, try + # them with a native, non-MSYS executable, such as `git config`. + test "$(git config -f file-link a.b)" = "c" && + test "$(git config -f dir-link/file2 x.y)" = "z" +' + +test_done diff --git a/t/t2300-cd-to-toplevel.sh b/t/t2300-cd-to-toplevel.sh index c8de6d8a190220..91f523d5198d8d 100755 --- a/t/t2300-cd-to-toplevel.sh +++ b/t/t2300-cd-to-toplevel.sh @@ -16,7 +16,7 @@ test_cd_to_toplevel () { test_expect_success $3 "$2" ' ( cd '"'$1'"' && - PATH="$EXEC_PATH:$PATH" && + PATH="$EXEC_PATH$PATH_SEP$PATH" && . git-sh-setup && cd_to_toplevel && [ "$(pwd -P)" = "$TOPLEVEL" ] diff --git a/t/t2403-worktree-move.sh b/t/t2403-worktree-move.sh index 0bb33e8b1b90fb..56faef26aa3bb1 100755 --- a/t/t2403-worktree-move.sh +++ b/t/t2403-worktree-move.sh @@ -271,4 +271,13 @@ test_expect_success 'move worktree with relative path to absolute path' ' test_cmp expect .git/worktrees/absolute/gitdir ' +test_expect_success MINGW 'worktree remove does not traverse mount points' ' + mkdir target && + >target/dont-remove-me && + git worktree add --detach wt-junction && + cmd //c "mklink /j wt-junction\\mnt target" && + git worktree remove --force wt-junction && + test_path_is_file target/dont-remove-me +' + test_done diff --git a/t/t3307-notes-man.sh b/t/t3307-notes-man.sh index 1aa366a410e9a3..7e5c06e6615d7a 100755 --- a/t/t3307-notes-man.sh +++ b/t/t3307-notes-man.sh @@ -26,7 +26,7 @@ test_expect_success 'example 1: notes to add an Acked-by line' ' ' test_expect_success 'example 2: binary notes' ' - cp "$TEST_DIRECTORY"/test-binary-1.png . && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png . && git checkout B && blob=$(git hash-object -w test-binary-1.png) && git notes --ref=logo add -C "$blob" && diff --git a/t/t3418-rebase-continue.sh b/t/t3418-rebase-continue.sh index f9b8999db50f1b..e03a28c0aaad24 100755 --- a/t/t3418-rebase-continue.sh +++ b/t/t3418-rebase-continue.sh @@ -82,7 +82,7 @@ test_expect_success 'rebase --continue remembers merge strategy and options' ' rm -f actual && ( - PATH=./test-bin:$PATH && + PATH=./test-bin$PATH_SEP$PATH && test_must_fail git rebase -s funny -X"option=arg with space" \ -Xop\"tion\\ -X"new${LF}line " main topic ) && @@ -91,7 +91,7 @@ test_expect_success 'rebase --continue remembers merge strategy and options' ' echo "Resolved" >F2 && git add F2 && ( - PATH=./test-bin:$PATH && + PATH=./test-bin$PATH_SEP$PATH && git rebase --continue ) && test_cmp expect actual diff --git a/t/t3700-add.sh b/t/t3700-add.sh index 2947bf9a6b1404..b9495e5cf00724 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -587,4 +587,15 @@ test_expect_success CASE_INSENSITIVE_FS 'path is case-insensitive' ' git add "$downcased" ' +test_expect_success MINGW 'can add files via NTFS junctions' ' + test_when_finished "cmd //c rmdir junction && rm -rf target" && + test_create_repo target && + cmd //c "mklink /j junction target" && + >target/via-junction && + git -C junction add "$(pwd)/junction/via-junction" && + echo via-junction >expect && + git -C target diff --cached --name-only >actual && + test_cmp expect actual +' + test_done diff --git a/t/t3701-add-interactive.sh b/t/t3701-add-interactive.sh index 6e120a40011238..cb09158c214768 100755 --- a/t/t3701-add-interactive.sh +++ b/t/t3701-add-interactive.sh @@ -1204,6 +1204,27 @@ test_expect_success 'checkout -p patch editing of added file' ' ) ' +test_expect_success EXPENSIVE 'add -i with a lot of files' ' + git reset --hard && + x160=0123456789012345678901234567890123456789 && + x160=$x160$x160$x160$x160 && + y= && + i=0 && + while test $i -le 200 + do + name=$(printf "%s%03d" $x160 $i) && + echo $name >$name && + git add -N $name && + y="${y}y$LF" && + i=$(($i+1)) || + exit 1 + done && + echo "$y" | git add -p -- . && + git diff --cached >staged && + test_line_count = 1407 staged && + git reset --hard +' + test_expect_success 'show help from add--helper' ' git reset --hard && cat >expect <<-EOF && diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index ecc35aae82a5fe..15534bbb757e0c 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -1378,7 +1378,7 @@ test_expect_success 'stash -- works with binary files' ' mkdir -p subdir && >subdir/untracked && >subdir/tracked && - cp "$TEST_DIRECTORY"/test-binary-1.png subdir/tracked-binary && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png subdir/tracked-binary && git add subdir/tracked* && git stash -- subdir/ && test_path_is_missing subdir/tracked && diff --git a/t/t4012-diff-binary.sh b/t/t4012-diff-binary.sh index 97b5ac04071d36..0fb50d2ffc91d9 100755 --- a/t/t4012-diff-binary.sh +++ b/t/t4012-diff-binary.sh @@ -19,7 +19,7 @@ test_expect_success 'prepare repository' ' echo AIT >a && echo BIT >b && echo CIT >c && echo DIT >d && git update-index --add a b c d && echo git >a && - cat "$TEST_DIRECTORY"/test-binary-1.png >b && + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >b && echo git >c && cat b b >d ' diff --git a/t/t4049-diff-stat-count.sh b/t/t4049-diff-stat-count.sh index eceb47c8594416..2161a1e8cf5ba6 100755 --- a/t/t4049-diff-stat-count.sh +++ b/t/t4049-diff-stat-count.sh @@ -33,7 +33,7 @@ test_expect_success 'binary changes do not count in lines' ' git reset --hard && echo a >a && echo c >c && - cat "$TEST_DIRECTORY"/test-binary-1.png >d && + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >d && cat >expect <<-\EOF && a | 1 + c | 1 + diff --git a/t/t4108-apply-threeway.sh b/t/t4108-apply-threeway.sh index f30e85659dbb87..7f84edd9653a7d 100755 --- a/t/t4108-apply-threeway.sh +++ b/t/t4108-apply-threeway.sh @@ -272,11 +272,11 @@ test_expect_success 'apply with --3way --cached and conflicts' ' test_expect_success 'apply binary file patch' ' git reset --hard main && - cp "$TEST_DIRECTORY/test-binary-1.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" bin.png && git add bin.png && git commit -m "add binary file" && - cp "$TEST_DIRECTORY/test-binary-2.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-2.png" bin.png && git diff --binary >bin.diff && git reset --hard && @@ -287,11 +287,11 @@ test_expect_success 'apply binary file patch' ' test_expect_success 'apply binary file patch with 3way' ' git reset --hard main && - cp "$TEST_DIRECTORY/test-binary-1.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" bin.png && git add bin.png && git commit -m "add binary file" && - cp "$TEST_DIRECTORY/test-binary-2.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-2.png" bin.png && git diff --binary >bin.diff && git reset --hard && @@ -302,11 +302,11 @@ test_expect_success 'apply binary file patch with 3way' ' test_expect_success 'apply full-index patch with 3way' ' git reset --hard main && - cp "$TEST_DIRECTORY/test-binary-1.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" bin.png && git add bin.png && git commit -m "add binary file" && - cp "$TEST_DIRECTORY/test-binary-2.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-2.png" bin.png && git diff --full-index >bin.diff && git reset --hard && diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index c8c1c5c06b6037..8f2a2cbc6b8103 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -88,7 +88,7 @@ test_expect_success \ 'mkdir a && echo simple textfile >a/a && mkdir a/bin && - cp /bin/sh a/bin && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" a/bin && printf "text\r" >a/text.cr && printf "text\r\n" >a/text.crlf && printf "text\n" >a/text.lf && diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index e592c0bcde91e9..187a5206e17758 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -951,8 +951,8 @@ test_expect_success '"remote show" does not show symbolic refs' ' ( cd three && git remote show origin >output && - ! grep "^ *HEAD$" < output && - ! grep -i stale < output + ! grep "^ *HEAD$" .git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && @@ -1170,8 +1170,8 @@ test_expect_success !WITH_BREAKING_CHANGES 'migrate a remote from named file in ( cd seven && git remote rm origin && - mkdir .git/branches && - echo "quux#foom" > .git/branches/origin && + mkdir -p .git/branches && + echo "quux#foom" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && test "$(git config remote.origin.url)" = "quux" && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 1b986349a86f3e..e0f7756432366e 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -963,8 +963,8 @@ test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches' ' mk_empty testrepo && git branch second $the_first_commit && git checkout second && - mkdir testrepo/.git/branches && - echo ".." > testrepo/.git/branches/branch1 && + mkdir -p testrepo/.git/branches && + echo ".." >testrepo/.git/branches/branch1 && ( cd testrepo && git fetch branch1 && @@ -977,8 +977,8 @@ test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches' ' test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches containing #' ' mk_empty testrepo && - mkdir testrepo/.git/branches && - echo "..#second" > testrepo/.git/branches/branch2 && + mkdir -p testrepo/.git/branches && + echo "..#second" >testrepo/.git/branches/branch2 && ( cd testrepo && git fetch branch2 && @@ -994,8 +994,8 @@ test_expect_success !WITH_BREAKING_CHANGES 'push with branches' ' git checkout second && test_when_finished "rm -rf .git/branches" && - mkdir .git/branches && - echo "testrepo" > .git/branches/branch1 && + mkdir -p .git/branches && + echo "testrepo" >.git/branches/branch1 && git push branch1 && ( @@ -1010,8 +1010,8 @@ test_expect_success !WITH_BREAKING_CHANGES 'push with branches containing #' ' mk_empty testrepo && test_when_finished "rm -rf .git/branches" && - mkdir .git/branches && - echo "testrepo#branch3" > .git/branches/branch2 && + mkdir -p .git/branches && + echo "testrepo#branch3" >.git/branches/branch2 && git push branch2 && ( @@ -1541,7 +1541,7 @@ EOF git init no-thin && git --git-dir=no-thin/.git config receive.unpacklimit 0 && git push no-thin/.git refs/heads/main:refs/heads/foo && - echo modified >> path1 && + echo modified >>path1 && git commit -am modified && git repack -adf && rcvpck="git receive-pack --reject-thin-pack-for-testing" && diff --git a/t/t5532-fetch-proxy.sh b/t/t5532-fetch-proxy.sh index 95d0f33b29531c..86fe5d8f752147 100755 --- a/t/t5532-fetch-proxy.sh +++ b/t/t5532-fetch-proxy.sh @@ -32,7 +32,7 @@ test_expect_success 'setup proxy script' ' write_script proxy <<-\EOF echo >&2 "proxying for $*" - cmd=$(./proxy-get-cmd) + cmd=$("$PERL_PATH" ./proxy-get-cmd) echo >&2 "Running $cmd" exec $cmd EOF diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index a7d475dd68dbd7..aba09dccfed1f3 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -793,4 +793,107 @@ test_expect_success SPNEGO 'http.emptyAuth=false skips Negotiate' ' test_line_count = 1 actual_401s ' +test_lazy_prereq NTLM 'curl --version | grep -q NTLM' + +test_expect_success NTLM 'access using NTLM auth' ' + test_when_finished "per_test_cleanup" && + + set_credential_reply get <<-EOF && + username=user + password=pwd + EOF + + test_config_global credential.helper test-helper && + test_must_fail env GIT_TRACE_CURL=1 git \ + ls-remote "$HTTPD_URL/ntlm_auth/repo.git" 2>err && + test_grep "allowNTLMAuth" err && + + # Can be enabled via config + GIT_TRACE_CURL=1 git -c http.$HTTPD_URL.allowNTLMAuth=true \ + ls-remote "$HTTPD_URL/ntlm_auth/repo.git" && + + # Or via credential helper responding with ntlm=allow + set_credential_reply get <<-EOF && + username=user + password=pwd + ntlm=allow + EOF + + git ls-remote "$HTTPD_URL/ntlm_auth/repo.git" +' + +test_lazy_prereq SPNEGO 'curl --version | grep -qi "SPNEGO\|GSS-API\|Kerberos\|negotiate"' + +test_expect_success SPNEGO 'http.emptyAuth=auto attempts Negotiate before credential_fill' ' + test_when_finished "per_test_cleanup" && + + set_credential_reply get <<-EOF && + username=alice + password=secret-passwd + EOF + + # Basic base64(alice:secret-passwd) + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== + EOF + + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=200 + id=default response=WWW-Authenticate: Negotiate + id=default response=WWW-Authenticate: Basic realm="example.com" + EOF + + test_config_global credential.helper test-helper && + GIT_TRACE_CURL="$TRASH_DIRECTORY/trace-auto" \ + git -c http.emptyAuth=auto \ + ls-remote "$HTTPD_URL/custom_auth/repo.git" && + + # In auto mode with a Negotiate+Basic server, there should be + # three 401 responses: (1) initial no-auth request, (2) empty-auth + # retry where Negotiate fails (no Kerberos ticket), (3) libcurl + # internal Negotiate retry. The fourth attempt uses Basic + # credentials from credential_fill and succeeds. + grep "HTTP/[0-9.]* 401" "$TRASH_DIRECTORY/trace-auto" >actual_401s && + test_line_count = 3 actual_401s && + + expect_credential_query get <<-EOF + capability[]=authtype + capability[]=state + protocol=http + host=$HTTPD_DEST + wwwauth[]=Negotiate + wwwauth[]=Basic realm="example.com" + EOF +' + +test_expect_success SPNEGO 'http.emptyAuth=false skips Negotiate' ' + test_when_finished "per_test_cleanup" && + + set_credential_reply get <<-EOF && + username=alice + password=secret-passwd + EOF + + # Basic base64(alice:secret-passwd) + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== + EOF + + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=200 + id=default response=WWW-Authenticate: Negotiate + id=default response=WWW-Authenticate: Basic realm="example.com" + EOF + + test_config_global credential.helper test-helper && + GIT_TRACE_CURL="$TRASH_DIRECTORY/trace-false" \ + git -c http.emptyAuth=false \ + ls-remote "$HTTPD_URL/custom_auth/repo.git" && + + # With emptyAuth=false, Negotiate is stripped immediately and + # credential_fill is called right away. Only one 401 response. + grep "HTTP/[0-9.]* 401" "$TRASH_DIRECTORY/trace-false" >actual_401s && + test_line_count = 1 actual_401s +' + test_done diff --git a/t/t5571-pre-push-hook.sh b/t/t5571-pre-push-hook.sh index a11b20e378223e..25b8d50c9428a7 100755 --- a/t/t5571-pre-push-hook.sh +++ b/t/t5571-pre-push-hook.sh @@ -138,4 +138,16 @@ test_expect_success 'sigpipe does not cause pre-push hook failure' ' git push parent1 "refs/heads/b/*:refs/heads/b/*" ' +test_expect_success 'can write to stderr' ' + test_hook --clobber pre-push <<-\EOF && + echo foo >/dev/stderr && + exit 0 + EOF + + test_commit third && + echo foo >expect && + git push --quiet parent1 HEAD 2>actual && + test_cmp expect actual +' + test_done diff --git a/t/t5580-unc-paths.sh b/t/t5580-unc-paths.sh index 65ef1a3628ee94..e9df367d5777fd 100755 --- a/t/t5580-unc-paths.sh +++ b/t/t5580-unc-paths.sh @@ -20,14 +20,11 @@ fi UNCPATH="$(winpwd)" case "$UNCPATH" in [A-Z]:*) + WITHOUTDRIVE="${UNCPATH#?:}" # Use administrative share e.g. \\localhost\C$\git-sdk-64\usr\src\git # (we use forward slashes here because MSYS2 and Git accept them, and # they are easier on the eyes) - UNCPATH="//localhost/${UNCPATH%%:*}\$/${UNCPATH#?:}" - test -d "$UNCPATH" || { - skip_all='could not access administrative share; skipping' - test_done - } + UNCPATH="//localhost/${UNCPATH%%:*}\$$WITHOUTDRIVE" ;; *) skip_all='skipping UNC path tests, cannot determine current path as UNC' @@ -35,6 +32,18 @@ case "$UNCPATH" in ;; esac +test_expect_success 'clone into absolute path lacking a drive prefix' ' + USINGBACKSLASHES="$(echo "$WITHOUTDRIVE"/without-drive-prefix | + tr / \\\\)" && + git clone . "$USINGBACKSLASHES" && + test -f without-drive-prefix/.git/HEAD +' + +test -d "$UNCPATH" || { + skip_all='could not access administrative share; skipping' + test_done +} + test_expect_success setup ' test_commit initial ' diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 3dd229c1867244..4ec1435e52aae6 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -7,6 +7,16 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +# This test script contains test cases that need to create symbolic links. To +# make sure that these test cases are exercised in Git for Windows, where (for +# historical reasons) `ln -s` creates copies by default, let's specifically ask +# for `ln -s` to create symbolic links whenever possible. +if test_have_prereq MINGW +then + MSYS=${MSYS+$MSYS }winsymlinks:nativestrict + export MSYS +fi + X= test_have_prereq !MINGW || X=.exe @@ -78,6 +88,13 @@ test_expect_success 'clone respects GIT_WORK_TREE' ' ' +test_expect_success CASE_INSENSITIVE_FS 'core.worktree is not added due to path case' ' + + mkdir UPPERCASE && + git clone src "$(pwd)/uppercase" && + test "unset" = "$(git -C UPPERCASE config --default unset core.worktree)" +' + test_expect_success 'clone from hooks' ' test_create_repo r0 && diff --git a/t/t5605-clone-local.sh b/t/t5605-clone-local.sh index 2397f8fa618054..a7444acc5f89e4 100755 --- a/t/t5605-clone-local.sh +++ b/t/t5605-clone-local.sh @@ -11,6 +11,21 @@ repo_is_hardlinked() { test_line_count = 0 output } +if test_have_prereq MINGW,BUSYBOX +then + # BusyBox' `find` does not support `-links`. Besides, BusyBox-w32's + # lstat() does not report hard links, just like Git's mingw_lstat() + # (from where BusyBox-w32 got its initial implementation). + repo_is_hardlinked() { + for f in $(find "$1/objects" -type f) + do + "$SYSTEMROOT"/system32/fsutil.exe \ + hardlink list $f >links && + test_line_count -gt 1 links || return 1 + done + } +fi + test_expect_success 'preparing origin repository' ' : >file && git add . && git commit -m1 && git clone --bare . a.git && diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index 9d6aa2187f2aaa..1bfeccdeb49958 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -39,7 +39,7 @@ test_expect_success 'access alternate via absolute path' ' ' test_expect_success 'access multiple alternates' ' - check_obj "$PWD/one.git/objects:$PWD/two.git/objects" <<-EOF + check_obj "$PWD/one.git/objects$PATH_SEP$PWD/two.git/objects" <<-EOF $one blob $two blob EOF @@ -75,7 +75,7 @@ test_expect_success 'access alternate via relative path (subdir)' ' quoted='"one.git\057objects"' unquoted='two.git/objects' test_expect_success 'mix of quoted and unquoted alternates' ' - check_obj "$quoted:$unquoted" <<-EOF + check_obj "$quoted$PATH_SEP$unquoted" <<-EOF $one blob $two blob EOF diff --git a/t/t5801-remote-helpers.sh b/t/t5801-remote-helpers.sh index d21877150ed82e..3917da47276825 100755 --- a/t/t5801-remote-helpers.sh +++ b/t/t5801-remote-helpers.sh @@ -262,7 +262,7 @@ test_expect_success 'push update refs failure' ' echo "update fail" >>file && git commit -a -m "update fail" && git rev-parse --verify testgit/origin/heads/update >expect && - test_expect_code 1 env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ + test_must_fail env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ git push origin update && git rev-parse --verify testgit/origin/heads/update >actual && test_cmp expect actual diff --git a/t/t5802-connect-helper.sh b/t/t5802-connect-helper.sh index a7be375bceb8d3..26cbcebf3b2b24 100755 --- a/t/t5802-connect-helper.sh +++ b/t/t5802-connect-helper.sh @@ -86,7 +86,7 @@ test_expect_success 'set up fake git-daemon' ' "$TRASH_DIRECTORY/remote" EOF export TRASH_DIRECTORY && - PATH=$TRASH_DIRECTORY:$PATH + PATH=$TRASH_DIRECTORY$PATH_SEP$PATH ' test_expect_success 'ext command can connect to git daemon (no vhost)' ' diff --git a/t/t5813-proto-disable-ssh.sh b/t/t5813-proto-disable-ssh.sh index 045e2fe6ce376a..c78581dc9f4a1e 100755 --- a/t/t5813-proto-disable-ssh.sh +++ b/t/t5813-proto-disable-ssh.sh @@ -15,8 +15,23 @@ test_expect_success 'setup repository to clone' ' ' test_proto "host:path" ssh "remote:repo.git" -test_proto "ssh://" ssh "ssh://remote$PWD/remote/repo.git" -test_proto "git+ssh://" ssh "git+ssh://remote$PWD/remote/repo.git" + +hostdir="$PWD" +if test_have_prereq MINGW && test "/${PWD#/}" != "$PWD" +then + case "$PWD" in + [A-Za-z]:/*) + hostdir="${PWD#?:}" + ;; + *) + skip_all="Unhandled PWD '$PWD'; skipping rest" + test_done + ;; + esac +fi + +test_proto "ssh://" ssh "ssh://remote$hostdir/remote/repo.git" +test_proto "git+ssh://" ssh "git+ssh://remote$hostdir/remote/repo.git" # Don't even bother setting up a "-remote" directory, as ssh would generally # complain about the bogus option rather than completing our request. Our diff --git a/t/t6403-merge-file.sh b/t/t6403-merge-file.sh index 801284cf8fcde5..cc39753ad45810 100755 --- a/t/t6403-merge-file.sh +++ b/t/t6403-merge-file.sh @@ -355,12 +355,12 @@ test_expect_success "expected conflict markers" ' test_expect_success 'binary files cannot be merged' ' test_must_fail git merge-file -p \ - orig.txt "$TEST_DIRECTORY"/test-binary-1.png new1.txt 2> merge.err && + orig.txt "$TEST_DIRECTORY"/lib-diff/test-binary-1.png new1.txt 2> merge.err && grep "Cannot merge binary files" merge.err ' test_expect_success 'binary files cannot be merged with --object-id' ' - cp "$TEST_DIRECTORY"/test-binary-1.png . && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png . && git add orig.txt new1.txt test-binary-1.png && test_must_fail git merge-file --object-id \ :orig.txt :test-binary-1.png :new1.txt 2> merge.err && diff --git a/t/t6407-merge-binary.sh b/t/t6407-merge-binary.sh index e8a28717cece32..2547f1d504a2c5 100755 --- a/t/t6407-merge-binary.sh +++ b/t/t6407-merge-binary.sh @@ -9,7 +9,7 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME test_expect_success setup ' - cat "$TEST_DIRECTORY"/test-binary-1.png >m && + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >m && git add m && git ls-files -s | sed -e "s/ 0 / 1 /" >E1 && test_tick && diff --git a/t/t7006-pager.sh b/t/t7006-pager.sh index 9717e825f0d7a5..e3aa496a286331 100755 --- a/t/t7006-pager.sh +++ b/t/t7006-pager.sh @@ -54,7 +54,7 @@ test_expect_success !MINGW,TTY 'LESS and LV envvars set by git-sh-setup' ' sane_unset LESS LV && PAGER="env >pager-env.out; wc" && export PAGER && - PATH="$(git --exec-path):$PATH" && + PATH="$(git --exec-path)$PATH_SEP$PATH" && export PATH && test_terminal sh -c ". git-sh-setup && git_pager" ) && @@ -388,7 +388,7 @@ test_default_pager() { EOF chmod +x \$less && ( - PATH=.:\$PATH && + PATH=.$PATH_SEP\$PATH && export PATH && $full_command ) && diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh new file mode 100755 index 00000000000000..b7cbcbf869296c --- /dev/null +++ b/t/t7108-reset-stdin.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='reset --stdin' + +. ./test-lib.sh + +test_expect_success 'reset --stdin' ' + test_commit hello && + git rm hello.t && + test -z "$(git ls-files hello.t)" && + echo hello.t | git reset --stdin && + test hello.t = "$(git ls-files hello.t)" +' + +test_expect_success 'reset --stdin -z' ' + test_commit world && + git rm hello.t world.t && + test -z "$(git ls-files hello.t world.t)" && + printf world.tQworld.tQhello.tQ | q_to_nul | git reset --stdin -z && + printf "hello.t\nworld.t\n" >expect && + git ls-files >actual && + test_cmp expect actual +' + +test_expect_success '--stdin requires --mixed' ' + echo hello.t >list && + test_must_fail git reset --soft --stdin test.txt && + git add test.txt && + git commit -m A && + echo B >> test.txt && + git checkout . && + test -z "$(git status -s)" && + echo A > expect.txt && + test_cmp expect.txt test.txt && + cd .. && + rm -rf fscache-test +' + +test_expect_success MINGW 'fscache flush cache dir' ' + + git init fscache-test && + cd fscache-test && + git config core.fscache 1 && + echo A > test.txt && + git add test.txt && + git commit -m A && + rm test.txt && + mkdir test.txt && + touch test.txt/test.txt && + git checkout . && + test -z "$(git status -s)" && + echo A > expect.txt && + test_cmp expect.txt test.txt && + cd .. && + rm -rf fscache-test +' + test_expect_success setup ' fill x y z >same && fill 1 2 3 4 5 6 7 8 >one && diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index 00d4070156243b..6f16f3893191e7 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -800,4 +800,14 @@ test_expect_success 'traverse into directories that may have ignored entries' ' ) ' +test_expect_success MINGW 'clean does not traverse mount points' ' + mkdir target && + >target/dont-clean-me && + git init with-mountpoint && + cmd //c "mklink /j with-mountpoint\\mountpoint target" && + git -C with-mountpoint clean -dfx && + test_path_is_missing with-mountpoint/mountpoint && + test_path_is_file target/dont-clean-me +' + test_done diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh index 6abb00876a3372..a15bedb9448f60 100755 --- a/t/t7406-submodule-update.sh +++ b/t/t7406-submodule-update.sh @@ -14,6 +14,15 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +# This test script contains test cases that need to create symbolic links. To +# make sure that these test cases are exercised in Git for Windows, where (for +# historical reasons) `ln -s` creates copies by default, let's specifically ask +# for `ln -s` to create symbolic links whenever possible. +if test_have_prereq MINGW +then + MSYS=${MSYS+$MSYS }winsymlinks:nativestrict + export MSYS +fi compare_head() { diff --git a/t/t7429-submodule-long-path.sh b/t/t7429-submodule-long-path.sh new file mode 100755 index 00000000000000..458519eafd6f03 --- /dev/null +++ b/t/t7429-submodule-long-path.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 + +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' + git config --global protocol.file.allow always && + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ + git -c init.defaultBranch=long init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git checkout -B long && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin long + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout long && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_success 'recursive submodule with a long path' ' + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ + git -c init.defaultBranch=long init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git checkout -B long && + git submodule add ../parent foo && + git submodule update --init --recursive && + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin long + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' + +test_done diff --git a/t/t7502-commit-porcelain.sh b/t/t7502-commit-porcelain.sh index 05f6da4ad98448..8a013669a5aa95 100755 --- a/t/t7502-commit-porcelain.sh +++ b/t/t7502-commit-porcelain.sh @@ -623,6 +623,48 @@ test_expect_success 'cleanup commit messages (scissors option,-F,-e, scissors on test_must_be_empty actual ' +test_expect_success 'helper-editor' ' + + write_script lf-to-crlf.sh <<-\EOF + sed "s/\$/Q/" <"$1" | tr Q "\\015" >"$1".new && + mv -f "$1".new "$1" + EOF +' + +test_expect_success 'cleanup commit messages (scissors option,-F,-e, CR/LF line endings)' ' + + test_config core.editor "\"$PWD/lf-to-crlf.sh\"" && + scissors="# ------------------------ >8 ------------------------" && + + test_write_lines >text \ + "# Keep this comment" "" " $scissors" \ + "# Keep this comment, too" "$scissors" \ + "# Remove this comment" "$scissors" \ + "Remove this comment, too" && + + test_write_lines >expect \ + "# Keep this comment" "" " $scissors" \ + "# Keep this comment, too" && + + git commit --cleanup=scissors -e -F text --allow-empty && + git cat-file -p HEAD >raw && + sed -e "1,/^\$/d" raw >actual && + test_cmp expect actual +' + +test_expect_success 'cleanup commit messages (scissors option,-F,-e, scissors on first line, CR/LF line endings)' ' + + scissors="# ------------------------ >8 ------------------------" && + test_write_lines >text \ + "$scissors" \ + "# Remove this comment and any following lines" && + cp text /tmp/test2-text && + git commit --cleanup=scissors -e -F text --allow-empty --allow-empty-message && + git cat-file -p HEAD >raw && + sed -e "1,/^\$/d" raw >actual && + test_must_be_empty actual +' + test_expect_success 'cleanup commit messages (strip option,-F)' ' echo >>negative && diff --git a/t/t7606-merge-custom.sh b/t/t7606-merge-custom.sh index 81fb7c474c14c1..8197a1c46bb5b6 100755 --- a/t/t7606-merge-custom.sh +++ b/t/t7606-merge-custom.sh @@ -23,7 +23,7 @@ test_expect_success 'set up custom strategy' ' EOF chmod +x git-merge-theirs && - PATH=.:$PATH && + PATH=.$PATH_SEP$PATH && export PATH ' diff --git a/t/t7811-grep-open.sh b/t/t7811-grep-open.sh index 3160be59fd2e26..1a98d733dceb86 100755 --- a/t/t7811-grep-open.sh +++ b/t/t7811-grep-open.sh @@ -52,7 +52,7 @@ test_expect_success SIMPLEPAGER 'git grep -O' ' EOF echo grep.h >expect.notless && - PATH=.:$PATH git grep -O GREP_PATTERN >out && + PATH=.$PATH_SEP$PATH git grep -O GREP_PATTERN >out && { test_cmp expect.less pager-args || test_cmp expect.notless pager-args diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh new file mode 100755 index 00000000000000..1ba48cc47e1b35 --- /dev/null +++ b/t/t8100-git-survey.sh @@ -0,0 +1,108 @@ +#!/bin/sh + +test_description='git survey' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +TEST_PASSES_SANITIZE_LEAK=0 +export TEST_PASSES_SANITIZE_LEAK + +. ./test-lib.sh + +test_expect_success 'git survey -h shows experimental warning' ' + test_expect_code 129 git survey -h >usage && + grep "EXPERIMENTAL!" usage +' + +test_expect_success 'create a semi-interesting repo' ' + test_commit_bulk 10 && + git tag -a -m one one HEAD~5 && + git tag -a -m two two HEAD~3 && + git tag -a -m three three two && + git tag -a -m four four three && + git update-ref -d refs/tags/three && + git update-ref -d refs/tags/two +' + +test_expect_success 'git survey --progress' ' + GIT_PROGRESS_DELAY=0 git survey --all-refs --progress >out 2>err && + grep "Preparing object walk" err +' + +approximate_sizes() { + # very simplistic approximate rounding + sed -Ee "s/ *(1[0-9][0-9])( |$)/ ~0.1kB\2/g" \ + -e "s/ *(4[6-9][0-9]|5[0-6][0-9])( |$)/ ~0.5kB\2/g" \ + -e "s/ *(5[6-9][0-9]|6[0-6][0-9])( |$)/ ~0.6kB\2/g" \ + -e "s/ *1(4[89][0-9]|5[0-8][0-9])( |$)/ ~1.5kB\2/g" \ + -e "s/ *1(69[0-9]|7[0-9][0-9])( |$)/ ~1.7kB\2/g" \ + -e "s/ *1(79[0-9]|8[0-9][0-9])( |$)/ ~1.8kB\2/g" \ + -e "s/ *2(1[0-9][0-9]|20[0-1])( |$)/ ~2.1kB\2/g" \ + -e "s/ *2(3[0-9][0-9]|4[0-1][0-9])( |$)/ ~2.3kB\2/g" \ + -e "s/ *2(5[0-9][0-9]|6[0-1][0-9])( |$)/ ~2.5kB\2/g" \ + "$@" +} + +test_expect_success 'git survey (default)' ' + git survey --all-refs >out 2>err && + test_line_count = 0 err && + + test_oid_cache <<-EOF && + commits_sizes sha1:~1.5kB | ~2.1kB + commits_sizes sha256:~1.8kB | ~2.5kB + trees_sizes sha1:~0.5kB | ~1.7kB + trees_sizes sha256:~0.6kB | ~2.3kB + blobs_sizes sha1:~0.1kB | ~0.1kB + blobs_sizes sha256:~0.1kB | ~0.1kB + tags_sizes sha1:~0.5kB | ~0.5kB + tags_sizes sha256:~0.5kB | ~0.6kB + EOF + + tr , " " >expect <<-EOF && + GIT SURVEY for "$(pwd)" + ----------------------------------------------------- + + REFERENCES SUMMARY + ======================== + , Ref Type | Count + -----------------+------ + , Branches | 1 + Remote refs | 0 + Tags (all) | 2 + Tags (annotated) | 2 + + REACHABLE OBJECT SUMMARY + ======================== + Object Type | Count + ------------+------ + Tags | 4 + Commits | 10 + Trees | 10 + Blobs | 10 + + TOTAL OBJECT SIZES BY TYPE + =============================================== + Object Type | Count | Disk Size | Inflated Size + ------------+-------+-----------+-------------- + Commits | 10 | $(test_oid commits_sizes) + Trees | 10 | $(test_oid trees_sizes) + Blobs | 10 | $(test_oid blobs_sizes) + Tags | 4 | $(test_oid tags_sizes) + EOF + + approximate_sizes out >out-edited && + lines=$(wc -l out-trimmed && + test_cmp expect out-trimmed && + + for type in "DIRECTORIES" "FILES" + do + for metric in "COUNT" "DISK SIZE" "INFLATED SIZE" + do + grep "TOP $type BY $metric" out || return 1 + done || return 1 + done +' + +test_done diff --git a/t/t9003-help-autocorrect.sh b/t/t9003-help-autocorrect.sh index 8da318d2b543da..c7a03aae697ac0 100755 --- a/t/t9003-help-autocorrect.sh +++ b/t/t9003-help-autocorrect.sh @@ -13,7 +13,7 @@ test_expect_success 'setup' ' echo distimdistim was called EOF - PATH="$PATH:." && + PATH="$PATH$PATH_SEP." && export PATH && git commit --allow-empty -m "a single log entry" && diff --git a/t/t9200-git-cvsexportcommit.sh b/t/t9200-git-cvsexportcommit.sh index 581cf3d28fc05b..e2a86d7a903ebd 100755 --- a/t/t9200-git-cvsexportcommit.sh +++ b/t/t9200-git-cvsexportcommit.sh @@ -11,6 +11,13 @@ if ! test_have_prereq PERL; then test_done fi +case "$PWD" in +*:*) + skip_all='cvs would get confused by the colon in `pwd`; skipping tests' + test_done + ;; +esac + if ! cvs version >/dev/null 2>&1 then skip_all='skipping git cvsexportcommit tests, cvs not found' @@ -57,8 +64,8 @@ test_expect_success 'New file' ' mkdir A B C D E F && echo hello1 >A/newfile1.txt && echo hello2 >B/newfile2.txt && - cp "$TEST_DIRECTORY"/test-binary-1.png C/newfile3.png && - cp "$TEST_DIRECTORY"/test-binary-1.png D/newfile4.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png C/newfile3.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png D/newfile4.png && git add A/newfile1.txt && git add B/newfile2.txt && git add C/newfile3.png && @@ -83,8 +90,8 @@ test_expect_success 'Remove two files, add two and update two' ' rm -f B/newfile2.txt && rm -f C/newfile3.png && echo Hello5 >E/newfile5.txt && - cp "$TEST_DIRECTORY"/test-binary-2.png D/newfile4.png && - cp "$TEST_DIRECTORY"/test-binary-1.png F/newfile6.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-2.png D/newfile4.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png F/newfile6.png && git add E/newfile5.txt && git add F/newfile6.png && git commit -a -m "Test: Remove, add and update" && @@ -172,7 +179,7 @@ test_expect_success 'New file with spaces in file name' ' mkdir "G g" && echo ok then >"G g/with spaces.txt" && git add "G g/with spaces.txt" && \ - cp "$TEST_DIRECTORY"/test-binary-1.png "G g/with spaces.png" && \ + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png "G g/with spaces.png" && \ git add "G g/with spaces.png" && git commit -a -m "With spaces" && id=$(git rev-list --max-count=1 HEAD) && @@ -184,7 +191,7 @@ test_expect_success 'New file with spaces in file name' ' test_expect_success 'Update file with spaces in file name' ' echo Ok then >>"G g/with spaces.txt" && - cat "$TEST_DIRECTORY"/test-binary-1.png >>"G g/with spaces.png" && \ + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >>"G g/with spaces.png" && \ git add "G g/with spaces.png" && git commit -a -m "Update with spaces" && id=$(git rev-list --max-count=1 HEAD) && @@ -209,7 +216,7 @@ test_expect_success !MINGW 'File with non-ascii file name' ' mkdir -p Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö && echo Foo >Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.txt && git add Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.txt && - cp "$TEST_DIRECTORY"/test-binary-1.png Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.png && git add Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.png && git commit -a -m "Går det så går det" && \ id=$(git rev-list --max-count=1 HEAD) && diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index 784d68b6e5006f..d4e2222e032e1d 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -1010,4 +1010,15 @@ test_expect_success GPG,RUST 'export and import of doubly signed commit' ' fi ' +cat > expected << EOF +reset refs/heads/master +from $(git rev-parse master) + +EOF + +test_expect_failure 'refs are updated even if no commits need to be exported' ' + git fast-export master..master > actual && + test_cmp expected actual +' + test_done diff --git a/t/t9800-git-p4-basic.sh b/t/t9800-git-p4-basic.sh index 0816763e46639c..b3dbd02961fae3 100755 --- a/t/t9800-git-p4-basic.sh +++ b/t/t9800-git-p4-basic.sh @@ -286,7 +286,7 @@ test_expect_success 'exit when p4 fails to produce marshaled output' ' EOF chmod 755 badp4dir/p4 && ( - PATH="$TRASH_DIRECTORY/badp4dir:$PATH" && + PATH="$TRASH_DIRECTORY/badp4dir$PATH_SEP$PATH" && export PATH && test_expect_code 1 git p4 clone --dest="$git" //depot >errs 2>&1 ) && diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index 28f61f08fb4cec..c3b0af08e4a060 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -139,12 +139,7 @@ invalid_variable_name='${foo.bar}' actual="$TRASH_DIRECTORY/actual" -if test_have_prereq MINGW -then - ROOT="$(pwd -W)" -else - ROOT="$(pwd)" -fi +ROOT="$(pwd)" test_expect_success 'setup for __git_find_repo_path/__gitdir tests' ' mkdir -p subdir/subsubdir && diff --git a/t/test-lib.sh b/t/test-lib.sh index 4a7357b547e77e..30a16818841865 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -40,6 +40,15 @@ case "${GIT_TEST_USE_SET_E:-false}" in ;; esac +# On Unix/Linux, the path separator is the colon, on other systems it +# may be different, though. On Windows, for example, it is a semicolon. +# If the PATH variable contains semicolons, it is pretty safe to assume +# that the path separator is a semicolon. +case "$PATH" in +*\;*) PATH_SEP=\; ;; +*) PATH_SEP=: ;; +esac + # Test the binaries we have just built. The tests are kept in # t/ subdirectory and are run in 'trash directory' subdirectory. if test -z "$TEST_DIRECTORY" @@ -1419,7 +1428,7 @@ then done done IFS=$OLDIFS - PATH=$GIT_VALGRIND/bin:$PATH + PATH=$GIT_VALGRIND/bin$PATH_SEP$PATH GIT_EXEC_PATH=$GIT_VALGRIND/bin export GIT_VALGRIND GIT_VALGRIND_MODE="$valgrind" @@ -1431,7 +1440,7 @@ elif test -n "$GIT_TEST_INSTALLED" then GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) || error "Cannot run git from $GIT_TEST_INSTALLED." - PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH + PATH=$GIT_TEST_INSTALLED$PATH_SEP$GIT_BUILD_DIR/t/helper$PATH_SEP$PATH GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH} else # normal case, use ../bin-wrappers only unless $with_dashes: if test -n "$no_bin_wrappers" @@ -1447,12 +1456,12 @@ else # normal case, use ../bin-wrappers only unless $with_dashes: fi with_dashes=t fi - PATH="$git_bin_dir:$PATH" + PATH="$git_bin_dir$PATH_SEP$PATH" fi GIT_EXEC_PATH=$GIT_BUILD_DIR if test -n "$with_dashes" then - PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH" + PATH="$GIT_BUILD_DIR$PATH_SEP$GIT_BUILD_DIR/t/helper$PATH_SEP$PATH" fi fi GIT_TEMPLATE_DIR="$GIT_TEST_TEMPLATE_DIR" @@ -1702,17 +1711,30 @@ Darwin) test_set_prereq EXECKEEPSPID ;; *MINGW*) - # Windows has its own (incompatible) sort and find - sort () { - /usr/bin/sort "$@" - } - find () { - /usr/bin/find "$@" - } - # git sees Windows-style pwd - pwd () { - builtin pwd -W - } + if test -x /usr/bin/sort + then + # Windows has its own (incompatible) sort; override + sort () { + /usr/bin/sort "$@" + } + fi + if test -x /usr/bin/find + then + # Windows has its own (incompatible) find; override + find () { + /usr/bin/find "$@" + } + fi + # On Windows, Git wants Windows paths. But /usr/bin/pwd spits out + # Unix-style paths. At least in Bash, we have a builtin pwd that + # understands the -W option to force "mixed" paths, i.e. with drive + # prefix but still with forward slashes. Let's use that, if available. + if type builtin >/dev/null 2>&1 + then + pwd () { + builtin pwd -W + } + fi # no POSIX permissions # backslashes in pathspec are converted to '/' # exec does not inherit the PID @@ -1722,6 +1744,12 @@ Darwin) test_set_prereq GREP_STRIPS_CR test_set_prereq WINDOWS GIT_TEST_CMP="GIT_DIR=/dev/null git diff --no-index --ignore-cr-at-eol --" + if ! type iconv >/dev/null 2>&1 + then + iconv () { + test-tool iconv "$@" + } + fi ;; *CYGWIN*) test_set_prereq POSIXPERM @@ -1901,6 +1929,10 @@ test_lazy_prereq UNZIP ' test $? -ne 127 ' +test_lazy_prereq BUSYBOX ' + case "$($SHELL --help 2>&1)" in *BusyBox*) true;; *) false;; esac +' + run_with_limited_cmdline () { (ulimit -s 128 && "$@") } diff --git a/t/unit-tests/u-mingw.c b/t/unit-tests/u-mingw.c new file mode 100644 index 00000000000000..cb74da5e793a33 --- /dev/null +++ b/t/unit-tests/u-mingw.c @@ -0,0 +1,72 @@ +#include "unit-test.h" + +#if defined(GIT_WINDOWS_NATIVE) && !defined(_UCRT) +#undef strerror +int errnos_contains(int); +static int errnos [53]={ + /* errnos in err_win_to_posix */ + EACCES, EBUSY, EEXIST, ERANGE, EIO, ENODEV, ENXIO, ENOEXEC, EINVAL, ENOENT, + EPIPE, ENAMETOOLONG, ENOSYS, ENOTEMPTY, ENOSPC, EFAULT, EBADF, EPERM, EINTR, + E2BIG, ESPIPE, ENOMEM, EXDEV, EAGAIN, ENFILE, EMFILE, ECHILD, EROFS, + /* errnos only in winsock_error_to_errno */ + EWOULDBLOCK, EINPROGRESS, EALREADY, ENOTSOCK, EDESTADDRREQ, EMSGSIZE, + EPROTOTYPE, ENOPROTOOPT, EPROTONOSUPPORT, EOPNOTSUPP, EAFNOSUPPORT, + EADDRINUSE, EADDRNOTAVAIL, ENETDOWN, ENETUNREACH, ENETRESET, ECONNABORTED, + ECONNRESET, ENOBUFS, EISCONN, ENOTCONN, ETIMEDOUT, ECONNREFUSED, ELOOP, + EHOSTUNREACH + }; + +int errnos_contains(int errnum) +{ + for(int i=0;i<53;i++) + if(errnos[i]==errnum) + return 1; + return 0; +} +#endif + +void test_mingw__no_strerror_shim_on_ucrt(void) +{ +#if defined(GIT_WINDOWS_NATIVE) && defined(_UCRT) + cl_assert_(strerror != mingw_strerror, + "mingw_strerror is unnescessary when building against UCRT"); +#else + cl_skip(); +#endif +} + +void test_mingw__strerror(void) +{ +#if defined(GIT_WINDOWS_NATIVE) && !defined(_UCRT) + for(int i=0;i<53;i++) + { + char *crt; + char *mingw; + mingw = mingw_strerror(errnos[i]); + crt = strerror(errnos[i]); + cl_assert_(!strcasestr(mingw, "unknown error"), + "mingw_strerror should know all errno values we care about"); + if(!strcasestr(crt, "unknown error")) + cl_assert_equal_s(crt,mingw); + } +#else + cl_skip(); +#endif +} + +void test_mingw__errno_translation(void) +{ +#if defined(GIT_WINDOWS_NATIVE) && !defined(_UCRT) + /* GetLastError() return values are currently defined from 0 to 15841, + testing up to 20000 covers some room for future expansion */ + for (int i=0;i<20000;i++) + { + if(i!=ERROR_SUCCESS) + cl_assert_(errnos_contains(err_win_to_posix(i)), + "all err_win_to_posix return values should be tested against mingw_strerror"); + /* ideally we'd test the same for winsock_error_to_errno, but it's static */ + } +#else + cl_skip(); +#endif +} diff --git a/thread-utils.c b/thread-utils.c index 374890e6b05b69..00e7e9192b3e0b 100644 --- a/thread-utils.c +++ b/thread-utils.c @@ -28,11 +28,28 @@ int online_cpus(void) #endif #ifdef GIT_WINDOWS_NATIVE - SYSTEM_INFO info; - GetSystemInfo(&info); - - if ((int)info.dwNumberOfProcessors > 0) - return (int)info.dwNumberOfProcessors; + DWORD len = 0; + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &len) && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + uint8_t *buf = malloc(len); + if (buf) { + if (GetLogicalProcessorInformationEx(RelationProcessorCore, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf, &len)) { + DWORD offset = 0; + int n_cores = 0; + while (offset < len) { + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (buf + offset); + offset += info->Size; + /* The threads within a core always share a single group. We need to count the bits in the mask to get a thread count. */ + for (KAFFINITY mask = info->Processor.GroupMask[0].Mask; mask; mask >>= 1) + n_cores += mask &1; + } + if (n_cores) { + free(buf); + return n_cores; + } + } + free(buf); + } + } #elif defined(hpux) || defined(__hpux) || defined(_hpux) struct pst_dynamic psd; diff --git a/transport-helper.c b/transport-helper.c index 04d55572a9b371..b8b6ef118a4218 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -22,6 +22,8 @@ #include "packfile.h" static int debug; +/* TODO: put somewhere sensible, e.g. git_transport_options? */ +static int auto_gc = 1; struct helper_data { char *name; @@ -499,10 +501,25 @@ static int get_exporter(struct transport *transport, for (size_t i = 0; i < revlist_args->nr; i++) strvec_push(&fastexport->args, revlist_args->items[i].string); + strvec_push(&fastexport->args, "--"); + fastexport->git_cmd = 1; return start_command(fastexport); } +static void check_helper_status(struct helper_data *data) +{ + int pid, status; + + pid = waitpid(data->helper->pid, &status, WNOHANG); + if (pid < 0) + die("Could not retrieve status of remote helper '%s'", + data->name); + if (pid > 0 && WIFEXITED(status)) + die("Remote helper '%s' died with %d", + data->name, WEXITSTATUS(status)); +} + static int fetch_with_import(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -539,6 +556,7 @@ static int fetch_with_import(struct transport *transport, if (finish_command(&fastimport)) die(_("error while running fast-import")); + check_helper_status(data); /* * The fast-import stream of a remote helper that advertises @@ -572,6 +590,13 @@ static int fetch_with_import(struct transport *transport, } } strbuf_release(&buf); + if (auto_gc) { + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "gc", "--auto", "--quiet", NULL); + run_command(&cmd); + } return 0; } @@ -1177,6 +1202,7 @@ static int push_refs_with_export(struct transport *transport, if (finish_command(&exporter)) die(_("error while running fast-export")); + check_helper_status(data); if (push_update_refs_status(data, remote_refs, flags)) return 1; diff --git a/unix-socket.c b/unix-socket.c index 8860203c3f46dc..1fa0cf6c15c721 100644 --- a/unix-socket.c +++ b/unix-socket.c @@ -84,7 +84,7 @@ int unix_stream_connect(const char *path, int disallow_chdir) struct unix_sockaddr_context ctx; if (unix_sockaddr_init(&sa, path, &ctx, disallow_chdir) < 0) - return -1; + goto fail; fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd < 0) goto fail; diff --git a/unpack-trees.c b/unpack-trees.c index 998a1e6dc70cae..ff2f0fe57d5359 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1823,7 +1823,9 @@ static void mark_new_skip_worktree(struct pattern_list *pl, * 2. Widen worktree according to sparse-checkout file. * Matched entries will have skip_wt_flag cleared (i.e. "in") */ + enable_fscache(istate->cache_nr); clear_ce_flags(istate, select_flag, skip_wt_flag, pl, show_progress); + disable_fscache(); } static void populate_from_existing_patterns(struct unpack_trees_options *o, diff --git a/wt-status.c b/wt-status.c index b17372390cf96c..2148a402c5ce6f 100644 --- a/wt-status.c +++ b/wt-status.c @@ -40,7 +40,7 @@ #define UF_DELAY_WARNING_IN_MS (2 * 1000) static const char cut_line[] = -"------------------------ >8 ------------------------\n"; +"------------------------ >8 ------------------------"; static char default_wt_status_colors[][COLOR_MAXLEN] = { GIT_COLOR_NORMAL, /* WT_STATUS_HEADER */ @@ -1121,15 +1121,22 @@ static void wt_longstatus_print_other(struct wt_status *s, status_printf_ln(s, GIT_COLOR_NORMAL, "%s", ""); } +static inline int starts_with_newline(const char *p) +{ + return *p == '\n' || (*p == '\r' && p[1] == '\n'); +} + size_t wt_status_locate_end(const char *s, size_t len) { const char *p; struct strbuf pattern = STRBUF_INIT; strbuf_addf(&pattern, "\n%s %s", comment_line_str, cut_line); - if (starts_with(s, pattern.buf + 1)) + if (starts_with(s, pattern.buf + 1) && + starts_with_newline(s + pattern.len - 1)) len = 0; - else if ((p = strstr(s, pattern.buf))) { + else if ((p = strstr(s, pattern.buf)) && + starts_with_newline(p + pattern.len)) { size_t newlen = p - s + 1; if (newlen < len) len = newlen;