pytorch_connectomics/tutorials/waterz_decoding_large.yaml at master · PytorchConnectomics/pytorch_connectomics · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
experiment_name: waterz_decode_large
description: >
  Large-volume parallel waterz decoding using the file-backed orchestrator.

  Two modes:
    - overlap=[0,0,0] (default): non-overlapping chunks, single-face IOU stitching
    - overlap=[N,N,N]: overlapping chunks, fragment consensus + region graph merge

  Usage (serial):
    python scripts/decode_large.py --config tutorials/waterz_decoding_large.yaml

  Usage (parallel — launch N workers on SLURM):
    # Step 1: Initialize workflow (creates tasks on disk)
    python scripts/decode_large.py --config tutorials/waterz_decoding_large.yaml --init-only

    # Step 2: Launch workers (each claims and executes tasks)
    sbatch --array=0-25 scripts/decode_large_worker.sh tutorials/waterz_decoding_large.yaml

    # Step 3: Wait for completion + assemble
    python scripts/decode_large.py --config tutorials/waterz_decoding_large.yaml --wait --assemble

# ── Chunked decoding config ──────────────────────────────────────────────────
large_decode:
  # Input affinity file (HDF5, dataset "main", shape (C,Z,Y,X))
  affinity_path: "/projects/weilab/dataset/liconn/mansour/DL288B_251222S_cond5_40x_12tiles_round1_fused_488_crop512x1024x1024_ffn_sharp_tta_x8_prediction_uint8.h5"       # SET THIS

  # Workflow directory (shared filesystem for parallel workers)
  workflow_root: "/projects/weilab/weidf/lib/pytorch_connectomics/outputs/neuron_liconn_mit/DL228B/"       # SET THIS

  # Chunk layout
  chunk_shape: [40, 2066, 2066]   # voxels per chunk (Z, Y, X)
  overlap: [0, 0, 0]              # overlap per axis; [8,8,8] for overlap pipeline

  # Waterz agglomeration parameters
  thresholds: [0.4]
  merge_function: aff85_his256
  aff_threshold_low: 0.1
  aff_threshold_high: 0.9
  border_threshold: 0.3
  channel_order: xyz
  use_aff_uint8: true            # uint8 affinities (4x less aff memory)
  use_seg_uint32: true           # uint32 segment IDs (2x less seg memory)

  # Fragment initialization (per chunk, overlap pipeline only)
  #compute_fragments: true        # 2D slice-by-slice mahotas watershed
  #seed_method: maxima_distance   # maxima_distance[-T], minima[-T], grid[-N]

  # Dust merge (per chunk, after agglomeration)
  dust_merge: true
  dust_merge_size: 1500
  dust_merge_affinity: 0.1
  dust_remove_size: 600

  # Border stitching / overlap merge (same params as face_merge_pairs)
  min_overlap: 10                  # min overlap pixels to consider a pair
  iou_threshold: 0.0              # full Jaccard IOU threshold (0=disabled)
  one_sided_threshold: 0.9        # overlap/min_size for small-into-large merge
  one_sided_min_size: 0           # min segment size in face for one-sided merge
  affinity_threshold: 0.0         # min boundary affinity (0=disabled)

  # Execution
  backend: multiprocess            # "multiprocess" or "slurm"
  num_workers: 1                   # local parallel workers (--parallel default)
  slurm:
    partition: short
    mem: 64G
    cpus_per_task: 1
    time: "12:00:00"

  # Output
  write_output: true
  output_path: ""                  # default: workflow_root/assembled.h5
  compression: gzip
  compression_level: 4