From 05f581bd139d51dacedaffde629f177635c28443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6kalp=20=C3=96zcan?= Date: Sun, 1 Mar 2026 20:14:14 +1100 Subject: [PATCH] Fix toggle mask compilation and evaluation with ored terms Optimisation: - Compute or_fields once per evaluation (not per block) - Add or_fields to toggle context struct and use it to skip OR processing in the hot loop when there are no OR toggle terms (the common case) - When there are OR fields, search terms by field_index (only for those fields) Rebuild --- distr/flecs.c | 83 +++++++++++++++++++++-- src/query/compiler/compiler.c | 16 +++-- src/query/engine/eval_toggle.c | 66 ++++++++++++++++++- src/query/types.h | 1 + test/query/project.json | 2 + test/query/src/Toggle.c | 116 +++++++++++++++++++++++++++++++++ test/query/src/main.c | 12 +++- 7 files changed, 281 insertions(+), 15 deletions(-) diff --git a/distr/flecs.c b/distr/flecs.c index 58e4143759..d012aeb658 100644 --- a/distr/flecs.c +++ b/distr/flecs.c @@ -1605,6 +1605,7 @@ typedef struct { int32_t cur; int32_t block_index; ecs_flags64_t block; + ecs_flags64_t or_fields; ecs_termset_t prev_set_fields; bool optional_not; bool has_bitset; @@ -80012,13 +80013,15 @@ int flecs_query_insert_toggle( continue; } + const ecs_flags64_t field_bit = 1llu << term->field_index; + /* Source matches, set flag */ if (term->oper == EcsNot) { - not_toggles |= (1llu << j); + not_toggles |= field_bit; } else if (term->oper == EcsOptional) { - optional_toggles |= (1llu << j); + optional_toggles |= field_bit; } else { - and_toggles |= (1llu << j); + and_toggles |= field_bit; } fields_done |= (1llu << j); @@ -80051,11 +80054,15 @@ int flecs_query_insert_toggle( * set, separate instructions let the query engine backtrack to get * the right results. */ if (optional_toggles) { + ecs_flags64_t optional_fields_processed = 0; for (j = i; j < term_count; j ++) { - uint64_t field_bit = 1ull << j; + const uint64_t field_bit = 1ull << terms[j].field_index; if (!(optional_toggles & field_bit)) { continue; } + if (optional_fields_processed & field_bit) { + continue; + } ecs_query_op_t op = {0}; op.kind = EcsQueryToggleOption; @@ -80063,6 +80070,8 @@ int flecs_query_insert_toggle( op.first.entity = field_bit; op.flags = cur.flags; flecs_query_op_insert(&op, ctx); + + optional_fields_processed |= field_bit; } } } @@ -85225,13 +85234,54 @@ static inline int32_t flecs_ctz64(uint64_t v) { #endif } +static inline +void flecs_query_apply_or_mask( + const ecs_query_t *query, + ecs_table_t *table, + int32_t block_index, + int32_t field_index, + ecs_flags64_t *mask, + bool *has_bitset) +{ + const ecs_term_t *terms = query->terms; + const int32_t term_count = query->term_count; + ecs_flags64_t block = 0; + bool chain_has_bitset = false; + + for (int32_t i = 0; i < term_count; i ++) { + if (terms[i].field_index != field_index) { + continue; + } + + ecs_id_t id = terms[i].id; + ecs_bitset_t *bs = flecs_table_get_toggle(table, id); + if (bs) { + ecs_assert((64 * block_index) < bs->size, ECS_INTERNAL_ERROR, NULL); + block |= bs->data[block_index]; + chain_has_bitset = true; + } else if (ecs_table_has_id(query->world, table, id)) { + /* If a non-toggle component is present, it is always enabled + * for all entities in the table. */ + block = UINT64_MAX; + break; + } + } + + if (chain_has_bitset || block == UINT64_MAX) { + *mask &= block; + *has_bitset |= chain_has_bitset; + } +} + static flecs_query_row_mask_t flecs_query_get_row_mask( ecs_iter_t *it, + const ecs_query_t *query, ecs_table_t *table, int32_t block_index, ecs_flags64_t and_fields, ecs_flags64_t not_fields, + ecs_flags64_t or_fields, ecs_query_toggle_ctx_t *op_ctx) { ecs_flags64_t mask = UINT64_MAX; @@ -85253,6 +85303,12 @@ flecs_query_row_mask_t flecs_query_get_row_mask( ecs_abort(ECS_INTERNAL_ERROR, NULL); } + if ((or_fields & field_bit)) { + flecs_query_apply_or_mask(query, table, block_index, + i, &mask, &has_bitset); + continue; + } + ecs_id_t id = it->ids[i]; ecs_bitset_t *bs = flecs_table_get_toggle(table, id); if (!bs) { @@ -85288,7 +85344,7 @@ bool flecs_query_toggle_for_up( ecs_flags64_t fields = (and_fields | not_fields) & it->up_fields; for (i = 0; i < field_count; i ++) { - uint64_t field_bit = 1llu << i; + const uint64_t field_bit = 1llu << i; if (!(fields & field_bit)) { continue; } @@ -85402,7 +85458,8 @@ bool flecs_query_toggle_cmp( block_index = op_ctx->block_index = new_block_index; flecs_query_row_mask_t row_mask = flecs_query_get_row_mask( - it, table, block_index, and_fields, not_fields, op_ctx); + it, &ctx->query->pub, table, block_index, + and_fields, not_fields, op_ctx->or_fields, op_ctx); /* If table doesn't have bitset columns, all columns match */ if (!(op_ctx->has_bitset = row_mask.has_bitset)) { @@ -85494,6 +85551,18 @@ bool flecs_query_toggle( ecs_query_toggle_ctx_t *op_ctx = flecs_op_ctx(ctx, toggle); if (!redo) { op_ctx->prev_set_fields = it->set_fields; + + /* Precompute which fields have OR toggle terms */ + const ecs_query_t *q = &ctx->query->pub; + ecs_flags64_t or_fields = 0; + for (int32_t i = 0; i < q->term_count; i ++) { + if (q->terms[i].oper == EcsOr && + (q->terms[i].flags_ & EcsTermIsToggle)) + { + or_fields |= (1llu << q->terms[i].field_index); + } + } + op_ctx->or_fields = or_fields; } ecs_flags64_t and_fields = op->first.entity; @@ -85514,6 +85583,7 @@ bool flecs_query_toggle_option( op_ctx->prev_set_fields = it->set_fields; op_ctx->optional_not = false; op_ctx->has_bitset = false; + op_ctx->or_fields = 0; } repeat: {} @@ -85539,7 +85609,6 @@ repeat: {} return result; } - /** * @file query/engine/eval_trav.c * @brief Transitive/reflexive relationship traversal. diff --git a/src/query/compiler/compiler.c b/src/query/compiler/compiler.c index aabbeea1f0..70feb1fb05 100644 --- a/src/query/compiler/compiler.c +++ b/src/query/compiler/compiler.c @@ -770,13 +770,15 @@ int flecs_query_insert_toggle( continue; } + const ecs_flags64_t field_bit = 1llu << term->field_index; + /* Source matches, set flag */ if (term->oper == EcsNot) { - not_toggles |= (1llu << j); + not_toggles |= field_bit; } else if (term->oper == EcsOptional) { - optional_toggles |= (1llu << j); + optional_toggles |= field_bit; } else { - and_toggles |= (1llu << j); + and_toggles |= field_bit; } fields_done |= (1llu << j); @@ -809,11 +811,15 @@ int flecs_query_insert_toggle( * set, separate instructions let the query engine backtrack to get * the right results. */ if (optional_toggles) { + ecs_flags64_t optional_fields_processed = 0; for (j = i; j < term_count; j ++) { - uint64_t field_bit = 1ull << j; + const uint64_t field_bit = 1ull << terms[j].field_index; if (!(optional_toggles & field_bit)) { continue; } + if (optional_fields_processed & field_bit) { + continue; + } ecs_query_op_t op = {0}; op.kind = EcsQueryToggleOption; @@ -821,6 +827,8 @@ int flecs_query_insert_toggle( op.first.entity = field_bit; op.flags = cur.flags; flecs_query_op_insert(&op, ctx); + + optional_fields_processed |= field_bit; } } } diff --git a/src/query/engine/eval_toggle.c b/src/query/engine/eval_toggle.c index 43ffff6b2f..efbbde2d8f 100644 --- a/src/query/engine/eval_toggle.c +++ b/src/query/engine/eval_toggle.c @@ -28,13 +28,54 @@ static inline int32_t flecs_ctz64(uint64_t v) { #endif } +static inline +void flecs_query_apply_or_mask( + const ecs_query_t *query, + ecs_table_t *table, + int32_t block_index, + int32_t field_index, + ecs_flags64_t *mask, + bool *has_bitset) +{ + const ecs_term_t *terms = query->terms; + const int32_t term_count = query->term_count; + ecs_flags64_t block = 0; + bool chain_has_bitset = false; + + for (int32_t i = 0; i < term_count; i ++) { + if (terms[i].field_index != field_index) { + continue; + } + + ecs_id_t id = terms[i].id; + ecs_bitset_t *bs = flecs_table_get_toggle(table, id); + if (bs) { + ecs_assert((64 * block_index) < bs->size, ECS_INTERNAL_ERROR, NULL); + block |= bs->data[block_index]; + chain_has_bitset = true; + } else if (ecs_table_has_id(query->world, table, id)) { + /* If a non-toggle component is present, it is always enabled + * for all entities in the table. */ + block = UINT64_MAX; + break; + } + } + + if (chain_has_bitset || block == UINT64_MAX) { + *mask &= block; + *has_bitset |= chain_has_bitset; + } +} + static flecs_query_row_mask_t flecs_query_get_row_mask( ecs_iter_t *it, + const ecs_query_t *query, ecs_table_t *table, int32_t block_index, ecs_flags64_t and_fields, ecs_flags64_t not_fields, + ecs_flags64_t or_fields, ecs_query_toggle_ctx_t *op_ctx) { ecs_flags64_t mask = UINT64_MAX; @@ -56,6 +97,12 @@ flecs_query_row_mask_t flecs_query_get_row_mask( ecs_abort(ECS_INTERNAL_ERROR, NULL); } + if ((or_fields & field_bit)) { + flecs_query_apply_or_mask(query, table, block_index, + i, &mask, &has_bitset); + continue; + } + ecs_id_t id = it->ids[i]; ecs_bitset_t *bs = flecs_table_get_toggle(table, id); if (!bs) { @@ -91,7 +138,7 @@ bool flecs_query_toggle_for_up( ecs_flags64_t fields = (and_fields | not_fields) & it->up_fields; for (i = 0; i < field_count; i ++) { - uint64_t field_bit = 1llu << i; + const uint64_t field_bit = 1llu << i; if (!(fields & field_bit)) { continue; } @@ -205,7 +252,8 @@ bool flecs_query_toggle_cmp( block_index = op_ctx->block_index = new_block_index; flecs_query_row_mask_t row_mask = flecs_query_get_row_mask( - it, table, block_index, and_fields, not_fields, op_ctx); + it, &ctx->query->pub, table, block_index, + and_fields, not_fields, op_ctx->or_fields, op_ctx); /* If table doesn't have bitset columns, all columns match */ if (!(op_ctx->has_bitset = row_mask.has_bitset)) { @@ -297,6 +345,18 @@ bool flecs_query_toggle( ecs_query_toggle_ctx_t *op_ctx = flecs_op_ctx(ctx, toggle); if (!redo) { op_ctx->prev_set_fields = it->set_fields; + + /* Precompute which fields have OR toggle terms */ + const ecs_query_t *q = &ctx->query->pub; + ecs_flags64_t or_fields = 0; + for (int32_t i = 0; i < q->term_count; i ++) { + if (q->terms[i].oper == EcsOr && + (q->terms[i].flags_ & EcsTermIsToggle)) + { + or_fields |= (1llu << q->terms[i].field_index); + } + } + op_ctx->or_fields = or_fields; } ecs_flags64_t and_fields = op->first.entity; @@ -317,6 +377,7 @@ bool flecs_query_toggle_option( op_ctx->prev_set_fields = it->set_fields; op_ctx->optional_not = false; op_ctx->has_bitset = false; + op_ctx->or_fields = 0; } repeat: {} @@ -341,4 +402,3 @@ repeat: {} return result; } - diff --git a/src/query/types.h b/src/query/types.h index 9137a2ca94..9e0a670246 100644 --- a/src/query/types.h +++ b/src/query/types.h @@ -356,6 +356,7 @@ typedef struct { int32_t cur; int32_t block_index; ecs_flags64_t block; + ecs_flags64_t or_fields; ecs_termset_t prev_set_fields; bool optional_not; bool has_bitset; diff --git a/test/query/project.json b/test/query/project.json index 2e814e63ad..b298e8f407 100644 --- a/test/query/project.json +++ b/test/query/project.json @@ -2077,6 +2077,8 @@ "fixed_2_src_w_toggle", "this_w_fixed_src_w_toggle", "fixed_src_w_this_w_toggle", + "or_chain_3_toggleable_tags", + "or_chain_3_toggleable_components", "this_from_nothing", "this", "this_skip_initial", diff --git a/test/query/src/Toggle.c b/test/query/src/Toggle.c index 2669994d08..3a7fc0d225 100644 --- a/test/query/src/Toggle.c +++ b/test/query/src/Toggle.c @@ -6165,3 +6165,119 @@ void Toggle_toggle_0_src(void) { ecs_fini(world); } + +static +int32_t toggle_or_match_count( + ecs_world_t *world, + ecs_query_t *q) +{ + int32_t count = 0; + ecs_iter_t it = ecs_query_iter(world, q); + while (ecs_query_next(&it)) { + count += it.count; + } + return count; +} + +void Toggle_or_chain_3_toggleable_tags(void) { + ecs_world_t *world = ecs_mini(); + + ECS_TAG(world, TagA); + ECS_TAG(world, TagB); + ECS_TAG(world, TagC); + + ecs_add_id(world, TagA, EcsCanToggle); + ecs_add_id(world, TagB, EcsCanToggle); + ecs_add_id(world, TagC, EcsCanToggle); + + ecs_entity_t e = ecs_new_w_id(world, TagA); + ecs_add(world, e, TagB); + ecs_add(world, e, TagC); + + ecs_enable_id(world, e, TagA, false); + ecs_enable_id(world, e, TagB, false); + ecs_enable_id(world, e, TagC, false); + + ecs_query_t *q = ecs_query(world, { + .terms = { + { .id = TagA, .oper = EcsOr }, + { .id = TagB, .oper = EcsOr }, + { .id = TagC } + }, + .cache_kind = cache_kind + }); + test_assert(q != NULL); + + test_int(toggle_or_match_count(world, q), 0); + + ecs_enable_id(world, e, TagA, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_enable_id(world, e, TagA, false); + ecs_enable_id(world, e, TagB, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_enable_id(world, e, TagB, false); + ecs_enable_id(world, e, TagC, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_enable_id(world, e, TagA, true); + ecs_enable_id(world, e, TagB, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_query_fini(q); + + ecs_fini(world); +} + +void Toggle_or_chain_3_toggleable_components(void) { + ecs_world_t *world = ecs_mini(); + + ECS_COMPONENT(world, Position); + ECS_COMPONENT(world, Velocity); + ECS_COMPONENT(world, Mass); + + ecs_add_id(world, ecs_id(Position), EcsCanToggle); + ecs_add_id(world, ecs_id(Velocity), EcsCanToggle); + ecs_add_id(world, ecs_id(Mass), EcsCanToggle); + + ecs_entity_t e = ecs_new_w(world, Position); + ecs_set(world, e, Position, {10, 20}); + ecs_set(world, e, Velocity, {1, 2}); + ecs_set(world, e, Mass, {100}); + + ecs_enable_component(world, e, Position, false); + ecs_enable_component(world, e, Velocity, false); + ecs_enable_component(world, e, Mass, false); + + ecs_query_t *q = ecs_query(world, { + .terms = { + { .id = ecs_id(Position), .oper = EcsOr }, + { .id = ecs_id(Velocity), .oper = EcsOr }, + { .id = ecs_id(Mass) } + }, + .cache_kind = cache_kind + }); + test_assert(q != NULL); + + test_int(toggle_or_match_count(world, q), 0); + + ecs_enable_component(world, e, Position, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_enable_component(world, e, Position, false); + ecs_enable_component(world, e, Velocity, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_enable_component(world, e, Velocity, false); + ecs_enable_component(world, e, Mass, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_enable_component(world, e, Position, true); + ecs_enable_component(world, e, Velocity, true); + test_int(toggle_or_match_count(world, q), 1); + + ecs_query_fini(q); + + ecs_fini(world); +} diff --git a/test/query/src/main.c b/test/query/src/main.c index 699c6c74e7..b6fd135708 100644 --- a/test/query/src/main.c +++ b/test/query/src/main.c @@ -1999,6 +1999,8 @@ void Toggle_fixed_src_2_component_toggle(void); void Toggle_fixed_2_src_w_toggle(void); void Toggle_this_w_fixed_src_w_toggle(void); void Toggle_fixed_src_w_this_w_toggle(void); +void Toggle_or_chain_3_toggleable_tags(void); +void Toggle_or_chain_3_toggleable_components(void); void Toggle_this_from_nothing(void); void Toggle_this(void); void Toggle_this_skip_initial(void); @@ -10609,6 +10611,14 @@ bake_test_case Toggle_testcases[] = { "fixed_src_w_this_w_toggle", Toggle_fixed_src_w_this_w_toggle }, + { + "or_chain_3_toggleable_tags", + Toggle_or_chain_3_toggleable_tags + }, + { + "or_chain_3_toggleable_components", + Toggle_or_chain_3_toggleable_components + }, { "this_from_nothing", Toggle_this_from_nothing @@ -13944,7 +13954,7 @@ static bake_test_suite suites[] = { "Toggle", Toggle_setup, NULL, - 163, + 165, Toggle_testcases, 1, Toggle_params