tooling(scripts): classifiers consume list-workflow-paths.sh TSV output

Each classify-*.sh now auto-detects input format and emits the
sweep-target path as an explicit column.

Input formats (auto-detected from first byte):
  - JSONL (legacy):  {repo, path?, sha}  — from gh /search/code paginate
  - TSV (preferred): repo\tpath\tsha\tscope — from list-workflow-paths.sh

Output (all 4 classifiers, 7 columns):
  repo \t path \t sha \t class \t reason \t lines \t details

The path column carries the file's location inside the repo, so
sweeps can target nested copies as first-class wrapper sites (not
just root-level workflows). When path is empty (legacy JSONL without
.path field), it defaults to "".

Validated:
  - classify-mirror.sh /tmp/drift-survey/mirror-full.json (JSONL):
    267 TRIVIAL + 22 NEEDS_REVIEW, matches prior sweep-report.md.
  - classify-mirror.sh /tmp/drift-survey/scorecard-tuples.tsv (TSV):
    fetches blobs and emits per-(repo, path) rows correctly.

Shared `normalize_input` extracted into _lib.sh; each classifier
sources it. Reduces per-script duplication.

Refs: #199 (campaign), #204 (helper script).

Author: hyperpolymath

scripts/sweep-classifiers/_lib.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: PMPL-1.0-or-later
+# _lib.sh — Shared helpers for classify-*.sh scripts.
+#
+# Sourced by every classifier. Not directly executable.
+#
+# Provides: normalize_input <file>
+#
+#   Emits one TSV row per workflow file: "<repo>\t<path>\t<sha>".
+#   Accepts two input shapes auto-detected from the first line:
+#
+#     1. JSONL from `gh api /search/code` (legacy):
+#          {"repo":"foo","sha":"abc123"}
+#        Emitted path is empty (no nested-copy info available).
+#
+#     2. TSV from `list-workflow-paths.sh` (preferred — handles nested):
+#          foo<TAB><path><TAB>abc123<TAB>top-level|nested
+#        Emitted path is the original `<path>` value; the scope column
+#        is dropped.
+
+normalize_input() {
+  local file="$1"
+  if [[ "$(head -c1 "$file")" == "{" ]]; then
+    jq -r '[.repo, (.path // ""), .sha] | @tsv' "$file"
+  else
+    awk -F'\t' 'NF >= 3 { print $1 "\t" $2 "\t" $3 }' "$file"
+  fi
+}

scripts/sweep-classifiers/classify-codeql.sh

@@ -12,7 +12,11 @@
 #   NEEDS_REVIEW          — NONE language matrix, or large custom workflow
 
 set -euo pipefail
-REPOS_JSON="${1:-/tmp/drift-survey/codeql-full.json}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+INPUT="${1:-/tmp/drift-survey/codeql-full.json}"
 BLOBS_DIR="${BLOBS_DIR:-/tmp/drift-survey/codeql-blobs}"
 mkdir -p "$BLOBS_DIR"
 
@@ -37,10 +41,10 @@ classify_blob() {
 }
 
 echo "[fetch] retrieving unique blobs..." >&2
-jq -r '.sha' "$REPOS_JSON" | sort -u | grep -v '^null$' | while read -r sha; do
+normalize_input "$INPUT" | awk -F'\t' '{print $3 "\t" $1}' | sort -u | while IFS=$'\t' read -r sha repo; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && continue
   blob_file="$BLOBS_DIR/$sha.yml"
   [ -s "$blob_file" ] && continue
-  repo=$(jq -r --arg s "$sha" 'select(.sha == $s) | .repo' "$REPOS_JSON" | head -1)
   gh api "/repos/hyperpolymath/$repo/git/blobs/$sha" --jq '.content' 2>/dev/null \
     | base64 -d > "$blob_file" || echo "::warn fetch failed for $sha ($repo)" >&2
 done
@@ -53,7 +57,7 @@ for blob in "$BLOBS_DIR"/*.yml; do
   SHA_CLASS[$sha]=$(classify_blob "$blob")
 done
 
-jq -r '"\(.repo)\t\(.sha)"' "$REPOS_JSON" | while IFS=$'\t' read -r repo sha; do
-  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$sha"; continue; }
-  printf '%s\t%s\t%s\n' "$repo" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
+normalize_input "$INPUT" | while IFS=$'\t' read -r repo path sha; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$path" "$sha"; continue; }
+  printf '%s\t%s\t%s\t%s\n' "$repo" "$path" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
 done

scripts/sweep-classifiers/classify-hypatia-scan.sh

@@ -14,7 +14,11 @@
 #                              or self-source like hypatia/standards repos)
 
 set -euo pipefail
-REPOS_JSON="${1:-/tmp/drift-survey/hypatia-scan-full.json}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+INPUT="${1:-/tmp/drift-survey/hypatia-scan-full.json}"
 BLOBS_DIR="${BLOBS_DIR:-/tmp/drift-survey/hypatia-blobs}"
 mkdir -p "$BLOBS_DIR"
 
@@ -36,10 +40,10 @@ classify_blob() {
 }
 
 echo "[fetch] retrieving unique blobs..." >&2
-jq -r '.sha' "$REPOS_JSON" | sort -u | grep -v '^null$' | while read -r sha; do
+normalize_input "$INPUT" | awk -F'\t' '{print $3 "\t" $1}' | sort -u | while IFS=$'\t' read -r sha repo; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && continue
   blob_file="$BLOBS_DIR/$sha.yml"
   [ -s "$blob_file" ] && continue
-  repo=$(jq -r --arg s "$sha" 'select(.sha == $s) | .repo' "$REPOS_JSON" | head -1)
   gh api "/repos/hyperpolymath/$repo/git/blobs/$sha" --jq '.content' 2>/dev/null \
     | base64 -d > "$blob_file" || echo "::warn fetch failed for $sha ($repo)" >&2
 done
@@ -52,7 +56,7 @@ for blob in "$BLOBS_DIR"/*.yml; do
   SHA_CLASS[$sha]=$(classify_blob "$blob")
 done
 
-jq -r '"\(.repo)\t\(.sha)"' "$REPOS_JSON" | while IFS=$'\t' read -r repo sha; do
-  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$sha"; continue; }
-  printf '%s\t%s\t%s\n' "$repo" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
+normalize_input "$INPUT" | while IFS=$'\t' read -r repo path sha; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$path" "$sha"; continue; }
+  printf '%s\t%s\t%s\t%s\n' "$repo" "$path" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
 done

scripts/sweep-classifiers/classify-mirror.sh

@@ -17,7 +17,11 @@
 
 set -euo pipefail
 
-REPOS_JSON="${1:-/tmp/drift-survey/mirror-full.json}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+INPUT="${1:-/tmp/drift-survey/mirror-full.json}"
 BLOBS_DIR="${BLOBS_DIR:-/tmp/drift-survey/blobs}"
 mkdir -p "$BLOBS_DIR"
 
@@ -69,10 +73,10 @@ classify_blob() {
 
 # 1. Fetch each unique SHA's blob (cached).
 echo "[fetch] unique SHAs to retrieve..." >&2
-jq -r '.sha' "$REPOS_JSON" | sort -u | grep -v '^null$' | while read -r sha; do
+normalize_input "$INPUT" | awk -F'\t' '{print $3 "\t" $1}' | sort -u | while IFS=$'\t' read -r sha repo; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && continue
   blob_file="$BLOBS_DIR/$sha.yml"
   [ -s "$blob_file" ] && continue
-  repo=$(jq -r --arg s "$sha" 'select(.sha == $s) | .repo' "$REPOS_JSON" | head -1)
   gh api "/repos/hyperpolymath/$repo/git/blobs/$sha" --jq '.content' 2>/dev/null \
     | base64 -d > "$blob_file" || echo "::warn fetch failed for $sha ($repo)" >&2
 done
@@ -86,8 +90,8 @@ for blob in "$BLOBS_DIR"/*.yml; do
   SHA_CLASS[$sha]=$(classify_blob "$blob")
 done
 
-# 3. Emit per-repo TSV.
-jq -r '"\(.repo)\t\(.sha)"' "$REPOS_JSON" | while IFS=$'\t' read -r repo sha; do
-  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$sha"; continue; }
-  printf '%s\t%s\t%s\n' "$repo" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
+# 3. Emit per-(repo, path) TSV: repo \t path \t sha \t class \t reason \t lines \t forges
+normalize_input "$INPUT" | while IFS=$'\t' read -r repo path sha; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$path" "$sha"; continue; }
+  printf '%s\t%s\t%s\t%s\n' "$repo" "$path" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
 done

scripts/sweep-classifiers/classify-secret-scanner.sh

@@ -15,7 +15,11 @@
 
 set -euo pipefail
 
-REPOS_JSON="${1:-/tmp/drift-survey/secret-scanner-full.json}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=_lib.sh
+. "$SCRIPT_DIR/_lib.sh"
+
+INPUT="${1:-/tmp/drift-survey/secret-scanner-full.json}"
 BLOBS_DIR="${BLOBS_DIR:-/tmp/drift-survey/secret-blobs}"
 mkdir -p "$BLOBS_DIR"
 
@@ -50,10 +54,10 @@ classify_blob() {
 
 # 1. Fetch unique blobs (cached).
 echo "[fetch] retrieving unique blobs..." >&2
-jq -r '.sha' "$REPOS_JSON" | sort -u | grep -v '^null$' | while read -r sha; do
+normalize_input "$INPUT" | awk -F'\t' '{print $3 "\t" $1}' | sort -u | while IFS=$'\t' read -r sha repo; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && continue
   blob_file="$BLOBS_DIR/$sha.yml"
   [ -s "$blob_file" ] && continue
-  repo=$(jq -r --arg s "$sha" 'select(.sha == $s) | .repo' "$REPOS_JSON" | head -1)
   gh api "/repos/hyperpolymath/$repo/git/blobs/$sha" --jq '.content' 2>/dev/null \
     | base64 -d > "$blob_file" || echo "::warn fetch failed for $sha ($repo)" >&2
 done
@@ -67,8 +71,8 @@ for blob in "$BLOBS_DIR"/*.yml; do
   SHA_CLASS[$sha]=$(classify_blob "$blob")
 done
 
-# 3. Per-repo TSV.
-jq -r '"\(.repo)\t\(.sha)"' "$REPOS_JSON" | while IFS=$'\t' read -r repo sha; do
-  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$sha"; continue; }
-  printf '%s\t%s\t%s\n' "$repo" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
+# 3. Per-(repo, path) TSV.
+normalize_input "$INPUT" | while IFS=$'\t' read -r repo path sha; do
+  [ -z "$sha" ] || [ "$sha" = "null" ] && { printf '%s\t%s\t%s\tNEEDS_REVIEW\tnull_sha\t-\t-\n' "$repo" "$path" "$sha"; continue; }
+  printf '%s\t%s\t%s\t%s\n' "$repo" "$path" "$sha" "${SHA_CLASS[$sha]:-NEEDS_REVIEW	fetch_failed	-	-}"
 done