Skip to content

Commit 3aae43f

Browse files
committed
Revert "perf(extract): dual-ended work queue bounds concurrent giant parses"
This reverts commit 0475a880e5735ef5cb0b0d99532e552096e5a2d5. Measured on the linux kernel: peak RSS unchanged (23.95 vs 24.4 GB, within run noise) at +22 s wall -- the extraction peak turned out to be monotonic retained-tree/source accumulation, which is schedule-independent, not the concurrent giant transients this queue bounded. Net negative as measured; the retention-side fix is parked for post-release. Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
1 parent 0475a88 commit 3aae43f

1 file changed

Lines changed: 6 additions & 65 deletions

File tree

src/pipeline/pass_parallel.c

Lines changed: 6 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -687,22 +687,7 @@ typedef struct {
687687
CBMFileResult **result_cache;
688688
_Atomic int64_t *shared_ids;
689689
_Atomic int *cancelled;
690-
_Atomic int next_file_idx; /* legacy single-cursor (unused when claimed != NULL) */
691-
692-
/* Dual-ended work queue (peak-RSS control): sorted[] is size-DESCENDING;
693-
* a small band of workers (big_workers = f(worker count)) claims from the
694-
* BIG end while the rest claim from the SMALL end. The number of giant
695-
* parse working-sets in flight is structurally bounded by big_workers —
696-
* the old single biggest-first cursor put the 12 largest files of the
697-
* kernel into 12 workers at once (global RSS peak 24.4 GB at t=47s, a
698-
* ~14 GB transient above the retained floor). Giants still start at t=0
699-
* (LPT makespan preserved). Per-file claim flags make the crossover
700-
* race-free; hints are advisory scan starts. Output is schedule-
701-
* independent (content-canonical since the determinism fix). */
702-
_Atomic unsigned char *claimed;
703-
_Atomic int big_hint;
704-
_Atomic int small_hint;
705-
int big_workers;
690+
_Atomic int next_file_idx;
706691

707692
cbm_pkg_entries_t *pkg_entries; /* per-worker manifest arrays (separate allocation) */
708693

@@ -725,28 +710,6 @@ typedef struct {
725710
_Atomic int bp_futile;
726711
} extract_ctx_t;
727712

728-
/* Claim the next unclaimed file from one end of the dual-ended queue.
729-
* Returns the sorted[] index, or -1 when the queue is exhausted. */
730-
static int pp_claim_next(extract_ctx_t *ec, bool from_big) {
731-
if (from_big) {
732-
for (int i = atomic_load_explicit(&ec->big_hint, memory_order_relaxed);
733-
i < ec->file_count; i++) {
734-
if (!atomic_exchange_explicit(&ec->claimed[i], 1, memory_order_relaxed)) {
735-
atomic_store_explicit(&ec->big_hint, i + 1, memory_order_relaxed);
736-
return i;
737-
}
738-
}
739-
} else {
740-
for (int i = atomic_load_explicit(&ec->small_hint, memory_order_relaxed); i >= 0; i--) {
741-
if (!atomic_exchange_explicit(&ec->claimed[i], 1, memory_order_relaxed)) {
742-
atomic_store_explicit(&ec->small_hint, i - 1, memory_order_relaxed);
743-
return i;
744-
}
745-
}
746-
}
747-
return -1;
748-
}
749-
750713
/* Cap on the number of index.file_oversized WARN lines (the full list still goes
751714
* to the response/logfile — this only throttles the stderr noise). */
752715
enum { PP_OVERSIZED_WARN_MAX = 32 };
@@ -803,22 +766,12 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
803766
ws->local_gbuf = cbm_gbuf_new_shared_ids(ec->project_name, ec->repo_path, ec->shared_ids);
804767
}
805768

806-
/* Pull files: dual-ended claim queue (see extract_ctx_t), with the
807-
* legacy single cursor as fallback when claim flags are unavailable. */
808-
bool from_big = worker_id < ec->big_workers;
769+
/* Pull files from shared atomic counter */
809770
while (SKIP_ONE) {
810-
int sort_pos;
811-
if (ec->claimed) {
812-
sort_pos = pp_claim_next(ec, from_big);
813-
if (sort_pos < 0) {
814-
break;
815-
}
816-
} else {
817-
sort_pos = atomic_fetch_add_explicit(&ec->next_file_idx, SKIP_ONE,
818-
memory_order_relaxed);
819-
if (sort_pos >= ec->file_count) {
820-
break;
821-
}
771+
int sort_pos =
772+
atomic_fetch_add_explicit(&ec->next_file_idx, SKIP_ONE, memory_order_relaxed);
773+
if (sort_pos >= ec->file_count) {
774+
break;
822775
}
823776
if (atomic_load_explicit(ec->cancelled, memory_order_relaxed)) {
824777
break;
@@ -1163,16 +1116,6 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
11631116
};
11641117
atomic_init(&ec.next_worker_id, 0);
11651118
atomic_init(&ec.next_file_idx, 0);
1166-
ec.claimed = calloc((size_t)file_count, sizeof(_Atomic unsigned char));
1167-
atomic_init(&ec.big_hint, 0);
1168-
atomic_init(&ec.small_hint, file_count - 1);
1169-
ec.big_workers = worker_count / 6;
1170-
if (ec.big_workers < 1) {
1171-
ec.big_workers = 1;
1172-
}
1173-
if (ec.big_workers > 3) {
1174-
ec.big_workers = 3;
1175-
}
11761119
atomic_init(&ec.retained_bytes, 0);
11771120
atomic_init(&ec.retain_cap_warned, 0);
11781121
atomic_init(&ec.oversized_warned, 0);
@@ -1183,8 +1126,6 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
11831126
cbm_parallel_for_opts_t parallel_opts = {.max_workers = worker_count, .force_pthreads = false};
11841127
cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts);
11851128
CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count);
1186-
free((void *)ec.claimed);
1187-
ec.claimed = NULL;
11881129

11891130
/* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */
11901131
CBM_PROF_START(t_merge);

0 commit comments

Comments
 (0)