perf: add hash-cache, git-ls-files traversal, parallel dispatch, and Elixir sync-all

- scripts/lib/scan-cache.sh: SHA-256 hash cache so fix scripts skip
  unchanged files between runs (panic-attacker-style incremental scanning)
- scripts/lib/smart-files.sh: git ls-files helper with vendored dir
  exclusions (winget-pkgs, compiler-source, deps, node_modules)
- dispatch-runner.sh: --parallel N for concurrent fix execution,
  --dedup-repo to skip duplicate repo+category entries
- fix-http-to-https.sh: use git ls-files + scan cache instead of find
- fix-todo-markers.sh: use git ls-files + scan cache instead of find
- sync-all-parallel.exs: Elixir rewrite of sync-all.sh using BEAM
  Task.async_stream for 32-worker concurrent fetch/pull/push

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: Jonathan D.A. Jewell

scripts/dispatch-runner.sh

@@ -16,6 +16,8 @@
 #   --repo REPO       Only process entries for a specific repo
 #   --limit N         Process at most N entries
 #   --manifest PATH   Path to manifest (default: verisimdb-data/dispatch/pending.jsonl)
+#   --parallel N      Run up to N fix scripts concurrently (default: 1)
+#   --dedup-repo      Group entries by repo, run only first per repo+category
 
 set -euo pipefail
 
@@ -69,6 +71,8 @@ FILTER_TIER=""
 FILTER_STRATEGY=""
 FILTER_REPO=""
 LIMIT=0
+PARALLEL=1
+DEDUP_REPO=false
 
 # Counters
 TOTAL=0
@@ -86,6 +90,8 @@ while [[ $# -gt 0 ]]; do
         --repo)       FILTER_REPO="$2"; shift 2 ;;
         --limit)      LIMIT="$2"; shift 2 ;;
         --manifest)   MANIFEST_PATH="$2"; shift 2 ;;
+        --parallel)   PARALLEL="$2"; shift 2 ;;
+        --dedup-repo) DEDUP_REPO=true; shift ;;
         -h|--help)
             head -18 "$0" | tail -16
             exit 0
@@ -303,8 +309,47 @@ execute_entry() {
     esac
 }
 
+# --- Deduplication ---
+# When --dedup-repo is set, only process the first entry per repo+category combo.
+# This avoids running the same fix script on the same repo multiple times.
+declare -A SEEN_REPO_CAT=()
+
+should_skip_dedup() {
+    local line="$1"
+    if [[ "$DEDUP_REPO" != "true" ]]; then
+        return 1
+    fi
+    local repo category key
+    repo=$(echo "$line" | jq -r '.repo')
+    category=$(echo "$line" | jq -r '.category // "unknown"')
+    key="${repo}::${category}"
+    if [[ -n "${SEEN_REPO_CAT[$key]+x}" ]]; then
+        return 0
+    fi
+    SEEN_REPO_CAT["$key"]=1
+    return 1
+}
+
 # --- Main loop ---
 LINE_NUM=0
+ACTIVE_JOBS=0
+PARALLEL_TMPDIR=$(mktemp -d /tmp/dispatch-parallel-XXXXXX)
+
+# Parallel wrapper: run execute_entry in background, track jobs
+run_entry() {
+    local line="$1"
+    if [[ "$PARALLEL" -le 1 ]]; then
+        execute_entry "$line"
+    else
+        # Wait if we've hit the parallel limit
+        while [[ "$ACTIVE_JOBS" -ge "$PARALLEL" ]]; do
+            wait -n 2>/dev/null || true
+            ((ACTIVE_JOBS--)) || true
+        done
+        execute_entry "$line" &
+        ((ACTIVE_JOBS++)) || true
+    fi
+}
 
 while IFS= read -r line; do
     # Skip empty lines
@@ -333,6 +378,11 @@ while IFS= read -r line; do
         [[ "$entry_repo" != "$FILTER_REPO" ]] && continue
     fi
 
+    # Deduplicate by repo+category
+    if should_skip_dedup "$line"; then
+        continue
+    fi
+
     ((TOTAL++)) || true
 
     # Check limit
@@ -341,10 +391,16 @@ while IFS= read -r line; do
         break
     fi
 
-    execute_entry "$line"
+    run_entry "$line"
 
 done < "$MANIFEST_PATH"
 
+# Wait for all background jobs to finish
+if [[ "$PARALLEL" -gt 1 ]]; then
+    wait
+fi
+rm -rf "$PARALLEL_TMPDIR" 2>/dev/null || true
+
 # --- Summary ---
 echo ""
 echo "=== Summary ==="

scripts/fix-http-to-https.sh

@@ -6,44 +6,93 @@
 # Fixes PA014 InsecureProtocol findings where http:// is used.
 # Upgrades to https:// where safe to do so.
 #
+# Uses git ls-files for fast traversal and hash-based caching to skip
+# files unchanged since last scan.
+#
 # Usage: fix-http-to-https.sh <repo-path> <finding-json>
 
 set -euo pipefail
 
 REPO_PATH="${1:?Usage: $0 <repo-path> <finding-json>}"
 FINDING_JSON="${2:?Missing finding JSON file}"
 
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
 echo "=== HTTP to HTTPS Fix ==="
 echo "  Repo: $REPO_PATH"
 
 FIXED_COUNT=0
 SKIPPED_COUNT=0
 
+# Source scan cache if available
+if [[ -f "$SCRIPT_DIR/lib/scan-cache.sh" ]]; then
+    source "$SCRIPT_DIR/lib/scan-cache.sh"
+    cache_init "fix-http-to-https" "$REPO_PATH"
+    USE_CACHE=true
+else
+    USE_CACHE=false
+fi
+
 # Domains known to support HTTPS
 SAFE_DOMAINS="github.com|gitlab.com|bitbucket.org|crates.io|npmjs.com|pypi.org|rubygems.org|hex.pm|opam.ocaml.org|hackage.haskell.org|pkg.go.dev|docs.rs|wikipedia.org|stackoverflow.com|mozilla.org|w3.org|schema.org|creativecommons.org|opensource.org|spdx.org"
 
-# Find all text files (not binary, not .git)
+# File extensions to check
+EXTENSIONS=("*.sh" "*.yml" "*.yaml" "*.md" "*.adoc" "*.txt" "*.json" "*.toml" "*.cfg" "*.ini" "*.rs" "*.ex" "*.exs" "*.res" "*.js")
+
+# Use git ls-files if in a git repo, otherwise find
+list_files() {
+    if [[ -d "$REPO_PATH/.git" ]]; then
+        (cd "$REPO_PATH" && git ls-files -z "${EXTENSIONS[@]}" 2>/dev/null) | \
+            while IFS= read -r -d '' f; do
+                # Skip vendored/generated directories
+                case "$f" in
+                    */node_modules/*|*/target/*|*/.git/*|*/deps/*|*/vendor/*|*/_build/*) continue ;;
+                    */winget-pkgs/*|*/compiler-source/*|*/.lake/*|*/__pycache__/*) continue ;;
+                esac
+                [[ -f "$REPO_PATH/$f" ]] && printf '%s\0' "$REPO_PATH/$f"
+            done
+    else
+        find "$REPO_PATH" -type f \( -name "*.sh" -o -name "*.yml" -o -name "*.yaml" -o -name "*.md" -o -name "*.adoc" -o -name "*.txt" -o -name "*.json" -o -name "*.toml" -o -name "*.cfg" -o -name "*.ini" -o -name "*.rs" -o -name "*.ex" -o -name "*.exs" -o -name "*.res" -o -name "*.js" \) \
+            -not -path "*/.git/*" -not -path "*/target/*" -not -path "*/node_modules/*" \
+            -not -path "*/winget-pkgs/*" -not -path "*/compiler-source/*" \
+            -print0 2>/dev/null
+    fi
+}
+
 while IFS= read -r -d '' file; do
-    # Skip binary files
-    if file "$file" | grep -q "binary"; then
+    # Skip via hash cache
+    if [[ "$USE_CACHE" == "true" ]] && cache_unchanged "$file"; then
         continue
     fi
 
+    # Skip binary files (quick check via file extension — avoid slow `file` command)
+    case "$file" in
+        *.png|*.jpg|*.gif|*.ico|*.woff|*.woff2|*.ttf|*.eot|*.svg|*.pdf|*.zip|*.tar|*.gz) continue ;;
+    esac
+
     if grep -qP 'http://(?!localhost|127\.|0\.0\.0\.|192\.168\.|10\.|172\.(1[6-9]|2[0-9]|3[01])\.)' "$file" 2>/dev/null; then
         rel_path="${file#$REPO_PATH/}"
 
         # Only fix URLs for known HTTPS-capable domains
         if grep -qP "http://($SAFE_DOMAINS)" "$file" 2>/dev/null; then
             sed -i -E "s#http://($SAFE_DOMAINS)#https://\1#g" "$file" 2>/dev/null || true
-            count=$(grep -cP "http://($SAFE_DOMAINS)" "$file" 2>/dev/null || echo 0)
             echo "  FIXED $rel_path"
             ((FIXED_COUNT++)) || true
         else
             echo "  SKIP $rel_path — unknown domain (manual review needed)"
             ((SKIPPED_COUNT++)) || true
         fi
     fi
-done < <(find "$REPO_PATH" -type f \( -name "*.sh" -o -name "*.yml" -o -name "*.yaml" -o -name "*.md" -o -name "*.adoc" -o -name "*.txt" -o -name "*.json" -o -name "*.toml" -o -name "*.cfg" -o -name "*.ini" -o -name "*.rs" -o -name "*.ex" -o -name "*.exs" -o -name "*.res" -o -name "*.js" \) -not -path "*/\.git/*" -not -path "*/target/*" -not -path "*/node_modules/*" -print0 2>/dev/null)
+
+    # Update cache regardless of whether we fixed anything
+    [[ "$USE_CACHE" == "true" ]] && cache_update "$file"
+done < <(list_files)
+
+# Save cache
+if [[ "$USE_CACHE" == "true" ]]; then
+    cache_save
+    cache_stats
+fi
 
 echo ""
 echo "Fixed: $FIXED_COUNT file(s), Skipped: $SKIPPED_COUNT file(s)"

scripts/fix-todo-markers.sh

@@ -6,23 +6,66 @@
 # Fixes PA018-unchecked-todo findings.
 # Converts bare TODOs to structured FILL markers or resolves them.
 #
+# Uses git ls-files for fast traversal and hash-based caching to skip
+# files unchanged since last scan.
+#
 # Usage: fix-todo-markers.sh <repo-path> <finding-json>
 
 set -euo pipefail
 
 REPO_PATH="${1:?Usage: $0 <repo-path> <finding-json>}"
 FINDING_JSON="${2:?Missing finding JSON file}"
 
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
 echo "=== TODO/FIXME Marker Fix ==="
 echo "  Repo: $REPO_PATH"
 
 FIXED_COUNT=0
 TODO_COUNT=0
 
-# Find files with TODO/FIXME (excluding .git, target, node_modules, vendor)
+# Source scan cache if available
+if [[ -f "$SCRIPT_DIR/lib/scan-cache.sh" ]]; then
+    source "$SCRIPT_DIR/lib/scan-cache.sh"
+    cache_init "fix-todo-markers" "$REPO_PATH"
+    USE_CACHE=true
+else
+    USE_CACHE=false
+fi
+
+# File extensions to check
+EXTENSIONS=("*.rs" "*.ex" "*.exs" "*.res" "*.js" "*.sh" "*.bash" "*.idr" "*.lean" "*.hs" "*.ml" "*.mli" "*.zig" "*.ada" "*.adb" "*.ads" "justfile" "Makefile" "*.toml")
+
+# Use git ls-files if in a git repo
+list_files() {
+    if [[ -d "$REPO_PATH/.git" ]]; then
+        (cd "$REPO_PATH" && git ls-files -z "${EXTENSIONS[@]}" 2>/dev/null) | \
+            while IFS= read -r -d '' f; do
+                case "$f" in
+                    */node_modules/*|*/target/*|*/.git/*|*/deps/*|*/vendor/*|*/_build/*) continue ;;
+                    */winget-pkgs/*|*/compiler-source/*|*/.lake/*|*/.pack/*) continue ;;
+                    *.min.js|*.min.css) continue ;;
+                esac
+                [[ -f "$REPO_PATH/$f" ]] && printf '%s\0' "$REPO_PATH/$f"
+            done
+    else
+        find "$REPO_PATH" -type f \
+            -not -path "*/.git/*" -not -path "*/target/*" -not -path "*/node_modules/*" \
+            -not -path "*/_build/*" -not -path "*/vendor/*" -not -path "*/.pack/*" \
+            -not -path "*/.lake/*" -not -path "*/winget-pkgs/*" -not -path "*/compiler-source/*" \
+            -not -name "*.min.js" -not -name "*.min.css" \
+            \( -name "*.rs" -o -name "*.ex" -o -name "*.exs" -o -name "*.res" -o -name "*.js" \
+               -o -name "*.sh" -o -name "*.bash" -o -name "*.idr" -o -name "*.lean" \
+               -o -name "*.hs" -o -name "*.ml" -o -name "*.mli" -o -name "*.zig" \
+               -o -name "*.ada" -o -name "*.adb" -o -name "*.ads" \
+               -o -name "justfile" -o -name "Makefile" -o -name "*.toml" \) \
+            -print0 2>/dev/null
+    fi
+}
+
 while IFS= read -r -d '' file; do
-    # Skip binary files
-    if file "$file" | grep -q "binary"; then
+    # Skip via hash cache
+    if [[ "$USE_CACHE" == "true" ]] && cache_unchanged "$file"; then
         continue
     fi
 
@@ -35,22 +78,16 @@ while IFS= read -r -d '' file; do
         TODO_COUNT=$((TODO_COUNT + count))
         ((FIXED_COUNT++)) || true
     fi
-done < <(find "$REPO_PATH" -type f \
-    -not -path "*/\.git/*" \
-    -not -path "*/target/*" \
-    -not -path "*/node_modules/*" \
-    -not -path "*/_build/*" \
-    -not -path "*/vendor/*" \
-    -not -path "*/.pack/*" \
-    -not -path "*/.lake/*" \
-    -not -name "*.min.js" \
-    -not -name "*.min.css" \
-    \( -name "*.rs" -o -name "*.ex" -o -name "*.exs" -o -name "*.res" -o -name "*.js" \
-       -o -name "*.sh" -o -name "*.bash" -o -name "*.idr" -o -name "*.lean" \
-       -o -name "*.hs" -o -name "*.ml" -o -name "*.mli" -o -name "*.zig" \
-       -o -name "*.ada" -o -name "*.adb" -o -name "*.ads" \
-       -o -name "justfile" -o -name "Makefile" -o -name "*.toml" \) \
-    -print0 2>/dev/null)
+
+    # Update cache
+    [[ "$USE_CACHE" == "true" ]] && cache_update "$file"
+done < <(list_files)
+
+# Save cache
+if [[ "$USE_CACHE" == "true" ]]; then
+    cache_save
+    cache_stats
+fi
 
 echo ""
 if [[ "$FIXED_COUNT" -gt 0 ]]; then