feat!: hardlink deduplications

Cargo.toml

@@ -49,6 +49,7 @@ cli = ["clap/derive", "clap_complete", "clap-utilities", "json"]
 cli-completions = ["cli"]
 
 [dependencies]
+dashmap = "^6.1.0"
 pipe-trait = "^0.4.0"
 smart-default = "^0.7.1"
 derive_more = { version = "^2.0.1", features = ["full"] }
@@ -72,5 +73,6 @@ sysinfo = "^0.35.2"
 build-fs-tree = "^0.7.1"
 command-extra = "^1.0.0"
 maplit = "^1.0.2"
+normalize-path = "^0.2.1"
 pretty_assertions = "^1.4.1"
 rand = "^0.9.1"

README.md

@@ -50,13 +50,14 @@ The benchmark was generated by [a GitHub Workflow](https://github.com/KSXGitHub/
 * Very fast.
 * Relative comparison of separate files.
 * Extensible via the library crate or JSON interface.
+* Unbiased regarding hardlinks: All hardlinks are treated as equally real.
+* Optional hardlink detection and deduplication (would make `pdu` proportionally slower).
 * Optional progress report (would make `pdu` slightly slower).
 * Customize tree depth.
 * Customize chart size.
 
 ## Limitations
 
-* Ignorant of hard links: All hard links are counted as real files.
 * Do not follow symbolic links.
 * Do not differentiate filesystem: Mounted folders are counted as normal folders.
 * The runtime is optimized at the expense of binary size.

ci/github-actions/benchmark/matrix.ts

@@ -111,6 +111,17 @@ export const COMPETING_BENCHMARK_MATRIX: readonly CompetingBenchmarkCategory[] =
       ['du', '--count-links'],
     ],
   },
+  {
+    id: 'deduplicate-hardlinks',
+    pduCliArgs: ['--deduplicate-hardlinks'],
+    competitors: [
+      ['dust', '--no-progress'],
+      ['dua'],
+      ['ncdu', '-o', '/dev/stdout', '-0'],
+      ['gdu', '--non-interactive', '--no-progress'],
+      ['du'],
+    ],
+  },
   {
     id: 'top-down',
     pduCliArgs: ['--top-down'],

exports/completion.bash

@@ -23,7 +23,7 @@ _pdu() {
 
     case "${cmd}" in
         pdu)
-            opts="-h -V --json-input --json-output --bytes-format --top-down --align-right --quantity --depth --max-depth --width --total-width --column-width --min-ratio --no-sort --no-errors --silent-errors --progress --threads --help --version [FILES]..."
+            opts="-h -V --json-input --json-output --bytes-format --detect-links --dedupe-links --deduplicate-hardlinks --top-down --align-right --quantity --depth --max-depth --width --total-width --column-width --min-ratio --no-sort --no-errors --silent-errors --progress --threads --omit-json-shared-details --omit-json-shared-summary --help --version [FILES]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

exports/completion.elv

@@ -29,12 +29,17 @@ set edit:completion:arg-completer[pdu] = {|@words|
             cand --threads 'Set the maximum number of threads to spawn. Could be either "auto", "max", or a number'
             cand --json-input 'Read JSON data from stdin'
             cand --json-output 'Print JSON data instead of an ASCII chart'
+            cand --deduplicate-hardlinks 'Detect and subtract the sizes of hardlinks from their parent directory totals'
+            cand --detect-links 'Detect and subtract the sizes of hardlinks from their parent directory totals'
+            cand --dedupe-links 'Detect and subtract the sizes of hardlinks from their parent directory totals'
             cand --top-down 'Print the tree top-down instead of bottom-up'
             cand --align-right 'Set the root of the bars to the right'
             cand --no-sort 'Do not sort the branches in the tree'
             cand --silent-errors 'Prevent filesystem error messages from appearing in stderr'
             cand --no-errors 'Prevent filesystem error messages from appearing in stderr'
             cand --progress 'Report progress being made at the expense of performance'
+            cand --omit-json-shared-details 'Do not output `.shared.details` in the JSON output'
+            cand --omit-json-shared-summary 'Do not output `.shared.summary` in the JSON output'
             cand -h 'Print help (see more with ''--help'')'
             cand --help 'Print help (see more with ''--help'')'
             cand -V 'Print version'

exports/completion.fish

@@ -11,10 +11,13 @@ complete -c pdu -l min-ratio -d 'Minimal size proportion required to appear' -r
 complete -c pdu -l threads -d 'Set the maximum number of threads to spawn. Could be either "auto", "max", or a number' -r
 complete -c pdu -l json-input -d 'Read JSON data from stdin'
 complete -c pdu -l json-output -d 'Print JSON data instead of an ASCII chart'
+complete -c pdu -l deduplicate-hardlinks -l detect-links -l dedupe-links -d 'Detect and subtract the sizes of hardlinks from their parent directory totals'
 complete -c pdu -l top-down -d 'Print the tree top-down instead of bottom-up'
 complete -c pdu -l align-right -d 'Set the root of the bars to the right'
 complete -c pdu -l no-sort -d 'Do not sort the branches in the tree'
 complete -c pdu -l silent-errors -l no-errors -d 'Prevent filesystem error messages from appearing in stderr'
 complete -c pdu -l progress -d 'Report progress being made at the expense of performance'
+complete -c pdu -l omit-json-shared-details -d 'Do not output `.shared.details` in the JSON output'
+complete -c pdu -l omit-json-shared-summary -d 'Do not output `.shared.summary` in the JSON output'
 complete -c pdu -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c pdu -s V -l version -d 'Print version'

exports/completion.ps1

@@ -32,12 +32,17 @@ Register-ArgumentCompleter -Native -CommandName 'pdu' -ScriptBlock {
             [CompletionResult]::new('--threads', '--threads', [CompletionResultType]::ParameterName, 'Set the maximum number of threads to spawn. Could be either "auto", "max", or a number')
             [CompletionResult]::new('--json-input', '--json-input', [CompletionResultType]::ParameterName, 'Read JSON data from stdin')
             [CompletionResult]::new('--json-output', '--json-output', [CompletionResultType]::ParameterName, 'Print JSON data instead of an ASCII chart')
+            [CompletionResult]::new('--deduplicate-hardlinks', '--deduplicate-hardlinks', [CompletionResultType]::ParameterName, 'Detect and subtract the sizes of hardlinks from their parent directory totals')
+            [CompletionResult]::new('--detect-links', '--detect-links', [CompletionResultType]::ParameterName, 'Detect and subtract the sizes of hardlinks from their parent directory totals')
+            [CompletionResult]::new('--dedupe-links', '--dedupe-links', [CompletionResultType]::ParameterName, 'Detect and subtract the sizes of hardlinks from their parent directory totals')
             [CompletionResult]::new('--top-down', '--top-down', [CompletionResultType]::ParameterName, 'Print the tree top-down instead of bottom-up')
             [CompletionResult]::new('--align-right', '--align-right', [CompletionResultType]::ParameterName, 'Set the root of the bars to the right')
             [CompletionResult]::new('--no-sort', '--no-sort', [CompletionResultType]::ParameterName, 'Do not sort the branches in the tree')
             [CompletionResult]::new('--silent-errors', '--silent-errors', [CompletionResultType]::ParameterName, 'Prevent filesystem error messages from appearing in stderr')
             [CompletionResult]::new('--no-errors', '--no-errors', [CompletionResultType]::ParameterName, 'Prevent filesystem error messages from appearing in stderr')
             [CompletionResult]::new('--progress', '--progress', [CompletionResultType]::ParameterName, 'Report progress being made at the expense of performance')
+            [CompletionResult]::new('--omit-json-shared-details', '--omit-json-shared-details', [CompletionResultType]::ParameterName, 'Do not output `.shared.details` in the JSON output')
+            [CompletionResult]::new('--omit-json-shared-summary', '--omit-json-shared-summary', [CompletionResultType]::ParameterName, 'Do not output `.shared.summary` in the JSON output')
             [CompletionResult]::new('-h', '-h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', '--help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('-V', '-V ', [CompletionResultType]::ParameterName, 'Print version')

exports/completion.zsh

@@ -28,14 +28,19 @@ block-count\:"Count numbers of blocks"))' \
 '*--column-width=[Maximum widths of the tree column and width of the bar column]:TREE_WIDTH:_default:TREE_WIDTH:_default' \
 '--min-ratio=[Minimal size proportion required to appear]:MIN_RATIO:_default' \
 '--threads=[Set the maximum number of threads to spawn. Could be either "auto", "max", or a number]:THREADS:_default' \
-'(--quantity)--json-input[Read JSON data from stdin]' \
+'(--quantity --deduplicate-hardlinks)--json-input[Read JSON data from stdin]' \
 '--json-output[Print JSON data instead of an ASCII chart]' \
+'--deduplicate-hardlinks[Detect and subtract the sizes of hardlinks from their parent directory totals]' \
+'--detect-links[Detect and subtract the sizes of hardlinks from their parent directory totals]' \
+'--dedupe-links[Detect and subtract the sizes of hardlinks from their parent directory totals]' \
 '--top-down[Print the tree top-down instead of bottom-up]' \
 '--align-right[Set the root of the bars to the right]' \
 '--no-sort[Do not sort the branches in the tree]' \
 '--silent-errors[Prevent filesystem error messages from appearing in stderr]' \
 '--no-errors[Prevent filesystem error messages from appearing in stderr]' \
 '--progress[Report progress being made at the expense of performance]' \
+'--omit-json-shared-details[Do not output \`.shared.details\` in the JSON output]' \
+'--omit-json-shared-summary[Do not output \`.shared.summary\` in the JSON output]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '-V[Print version]' \