don't match chunks with hidden fragments (while they are not explicit)

Author: BlackMATov

develop/fuzzing/hidden_fuzz.lua

@@ -0,0 +1,81 @@
+local evo = require 'evolved'
+
+evo.debug_mode(true)
+
+---
+---
+---
+---
+---
+
+local __table_unpack = (function()
+    ---@diagnostic disable-next-line: deprecated
+    return table.unpack or unpack
+end)()
+
+---
+---
+---
+---
+---
+
+local all_entity_list = {} ---@type evolved.entity[]
+
+for i = 1, math.random(1, 10) do
+    local entity = evo.id()
+    all_entity_list[i] = entity
+end
+
+for _, entity in ipairs(all_entity_list) do
+    for _ = 0, math.random(0, #all_entity_list) do
+        local fragment = all_entity_list[math.random(1, #all_entity_list)]
+        evo.set(entity, fragment)
+    end
+
+    if math.random(1, 5) == 1 then
+        evo.set(entity, evo.HIDDEN)
+    end
+end
+
+---
+---
+---
+---
+---
+
+for _ = 1, 100 do
+    local include_set = {} ---@type table<evolved.entity, integer>
+    local include_list = {} ---@type evolved.entity[]
+    local include_count = 0
+
+    for _ = 1, math.random(1, #all_entity_list) do
+        local include = all_entity_list[math.random(1, #all_entity_list)]
+
+        if not include_set[include] then
+            include_count = include_count + 1
+            include_set[include] = include_count
+            include_list[include_count] = include
+        end
+    end
+
+    local q = evo.builder():include(__table_unpack(include_list)):spawn()
+
+    for chunk in evo.execute(q) do
+        local fragment_list, fragment_count = chunk:fragments()
+        for i = 1, fragment_count do
+            local fragment = fragment_list[i]
+            assert(include_set[fragment] or not evo.has(fragment, evo.HIDDEN))
+        end
+    end
+
+    evo.destroy(q)
+end
+
+---
+---
+---
+---
+---
+
+evo.destroy(__table_unpack(all_entity_list))
+evo.collect_garbage()

develop/untests.lua

@@ -6235,3 +6235,87 @@ do
         assert(evo.has(e, f3) and evo.get(e, f3) == 3)
     end
 end
+
+do
+    local f1, f2, f3 = evo.id(3)
+
+    evo.set(f2, evo.HIDDEN)
+
+    do
+        local p = evo.spawn { [f1] = 11, [f2] = 22, [f3] = 33 }
+        local e = evo.clone(p)
+
+        assert(evo.has(p, f1) and evo.get(p, f1) == 11)
+        assert(evo.has(p, f2) and evo.get(p, f2) == 22)
+        assert(evo.has(p, f3) and evo.get(p, f3) == 33)
+
+        assert(evo.has(e, f1) and evo.get(e, f1) == 11)
+        assert(not evo.has(e, f2) and evo.get(e, f2) == nil)
+        assert(evo.has(e, f3) and evo.get(e, f3) == 33)
+    end
+end
+
+do
+    local f1, f2 = evo.id(2)
+
+    local p = evo.builder():prefab():set(f1, 11):set(f2, 22):spawn()
+    local e = evo.clone(p)
+
+    do
+        local q = evo.builder():include(f1, f2):spawn()
+        local iter, state = evo.execute(q)
+        local chunk, entity_list, entity_count = iter(state)
+        assert(chunk and entity_list and entity_count)
+        assert(chunk == evo.chunk(f1, f2))
+        assert(entity_count == 1 and entity_list[1] == e)
+    end
+
+    do
+        local q = evo.builder():exclude(f1):spawn()
+
+        for c in evo.execute(q) do
+            local fs, fc = c:fragments()
+            for i = 1, fc do assert(not evo.has(fs[i], evo.HIDDEN)) end
+        end
+    end
+
+    do
+        local q = evo.builder():spawn()
+
+        for c in evo.execute(q) do
+            local fs, fc = c:fragments()
+            for i = 1, fc do assert(not evo.has(fs[i], evo.HIDDEN)) end
+        end
+    end
+end
+
+do
+    local f1, f2 = evo.id(2)
+
+    evo.set(f2, evo.HIDDEN)
+
+    local e1 = evo.builder():set(f1, 11):spawn()
+    local e2 = evo.builder():set(f1, 11):set(f2, 22):spawn()
+
+    do
+        local q = evo.builder():include(f1):spawn()
+        local iter, state = evo.execute(q)
+        local chunk, entity_list, entity_count = iter(state)
+        assert(chunk and entity_list and entity_count)
+        assert(chunk == evo.chunk(f1))
+        assert(entity_count == 1 and entity_list[1] == e1)
+        chunk, entity_list, entity_count = iter(state)
+        assert(not chunk and not entity_list and not entity_count)
+    end
+
+    do
+        local q = evo.builder():include(f1, f2):spawn()
+        local iter, state = evo.execute(q)
+        local chunk, entity_list, entity_count = iter(state)
+        assert(chunk and entity_list and entity_count)
+        assert(chunk == evo.chunk(f1, f2))
+        assert(entity_count == 1 and entity_list[1] == e2)
+        chunk, entity_list, entity_count = iter(state)
+        assert(not chunk and not entity_list and not entity_count)
+    end
+end

evolved.lua

@@ -124,6 +124,8 @@ local __query_sorted_excludes = {} ---@type table<evolved.query, evolved.assoc_l
 ---@field package __has_assign_hooks boolean
 ---@field package __has_insert_hooks boolean
 ---@field package __has_remove_hooks boolean
+---@field package __has_hidden_major boolean
+---@field package __has_hidden_minors boolean
 ---@field package __has_hidden_fragments boolean
 local __chunk_mt = {}
 __chunk_mt.__index = __chunk_mt
@@ -619,22 +621,18 @@ local function __execute_iterator(execute_state)
         local chunk_child_list = chunk.__child_list
         local chunk_child_count = chunk.__child_count
 
-        if exclude_set then
-            for i = 1, chunk_child_count do
-                local chunk_child = chunk_child_list[i]
-                local chunk_child_fragment = chunk_child.__fragment
+        for i = 1, chunk_child_count do
+            local chunk_child = chunk_child_list[i]
+            local chunk_child_fragment = chunk_child.__fragment
 
-                if not exclude_set[chunk_child_fragment] then
-                    chunk_stack_size = chunk_stack_size + 1
-                    chunk_stack[chunk_stack_size] = chunk_child
-                end
-            end
-        else
-            __lua_table_move(
-                chunk_child_list, 1, chunk_child_count,
-                chunk_stack_size + 1, chunk_stack)
+            local is_chunk_child_matched =
+                (not chunk_child.__has_hidden_major) and
+                (not exclude_set or not exclude_set[chunk_child_fragment])
 
-            chunk_stack_size = chunk_stack_size + chunk_child_count
+            if is_chunk_child_matched then
+                chunk_stack_size = chunk_stack_size + 1
+                chunk_stack[chunk_stack_size] = chunk_child
+            end
         end
 
         local chunk_entity_list = chunk.__entity_list
@@ -693,6 +691,12 @@ local __DESTROY_POLICY_REMOVE_FRAGMENT = __acquire_id()
 ---
 
 local __safe_tbls = {
+    ---@type evolved.entity[]
+    __EMPTY_ENTITY_LIST = __lua_setmetatable({}, {
+        __tostring = function() return 'empty entity list' end,
+        __newindex = function() __error_fmt 'attempt to modify empty entity list' end
+    }),
+
     ---@type table<evolved.fragment, integer>
     __EMPTY_FRAGMENT_SET = __lua_setmetatable({}, {
         __tostring = function() return 'empty fragment set' end,
@@ -1002,20 +1006,21 @@ local function __new_chunk(chunk_parent, chunk_fragment)
     ---@type evolved.fragment[]
     local chunk_component_fragments = {}
 
-    local has_setup_hooks = (chunk_parent and chunk_parent.__has_setup_hooks)
+    local has_setup_hooks = (chunk_parent ~= nil and chunk_parent.__has_setup_hooks)
         or __evolved_has_any(chunk_fragment, __DEFAULT, __DUPLICATE)
 
-    local has_assign_hooks = (chunk_parent and chunk_parent.__has_assign_hooks)
+    local has_assign_hooks = (chunk_parent ~= nil and chunk_parent.__has_assign_hooks)
         or __evolved_has_any(chunk_fragment, __ON_SET, __ON_ASSIGN)
 
-    local has_insert_hooks = (chunk_parent and chunk_parent.__has_insert_hooks)
+    local has_insert_hooks = (chunk_parent ~= nil and chunk_parent.__has_insert_hooks)
         or __evolved_has_any(chunk_fragment, __ON_SET, __ON_INSERT)
 
-    local has_remove_hooks = (chunk_parent and chunk_parent.__has_remove_hooks)
+    local has_remove_hooks = (chunk_parent ~= nil and chunk_parent.__has_remove_hooks)
         or __evolved_has(chunk_fragment, __ON_REMOVE)
 
-    local has_hidden_fragments = (chunk_parent and chunk_parent.__has_hidden_fragments)
-        or __evolved_has(chunk_fragment, __HIDDEN)
+    local has_hidden_major = __evolved_has(chunk_fragment, __HIDDEN)
+    local has_hidden_minors = chunk_parent ~= nil and chunk_parent.__has_hidden_fragments
+    local has_hidden_fragments = has_hidden_major or has_hidden_minors
 
     ---@type evolved.chunk
     local chunk = __lua_setmetatable({
@@ -1040,6 +1045,8 @@ local function __new_chunk(chunk_parent, chunk_fragment)
         __has_assign_hooks = has_assign_hooks,
         __has_insert_hooks = has_insert_hooks,
         __has_remove_hooks = has_remove_hooks,
+        __has_hidden_major = has_hidden_major,
+        __has_hidden_minors = has_hidden_minors,
         __has_hidden_fragments = has_hidden_fragments,
     }, __chunk_mt)
 
@@ -1237,18 +1244,26 @@ local function __chunk_without_hidden_fragments(chunk)
         return chunk
     end
 
-    local chunk_fragment_list = chunk.__fragment_list
-    local chunk_fragment_count = chunk.__fragment_count
+    while chunk and chunk.__has_hidden_major do
+        chunk = chunk.__parent
+    end
+
+    local new_chunk = nil
+
+    if chunk then
+        local chunk_fragment_list = chunk.__fragment_list
+        local chunk_fragment_count = chunk.__fragment_count
 
-    for i = chunk_fragment_count, 1, -1 do
-        local fragment = chunk_fragment_list[i]
+        for i = 1, chunk_fragment_count do
+            local fragment = chunk_fragment_list[i]
 
-        if __evolved_has(fragment, __HIDDEN) then
-            chunk = __chunk_without_fragment(chunk, fragment)
+            if not __evolved_has(fragment, __HIDDEN) then
+                new_chunk = __chunk_with_fragment(new_chunk, fragment)
+            end
         end
     end
 
-    return chunk
+    return new_chunk
 end
 
 ---
@@ -4329,6 +4344,7 @@ function __evolved_execute(query)
     local chunk_stack_size = 0
 
     local query_includes = __query_sorted_includes[query]
+    local query_include_set = query_includes and query_includes.__item_set --[[@as table<evolved.fragment, integer>]]
     local query_include_list = query_includes and query_includes.__item_list --[=[@as evolved.fragment[]]=]
     local query_include_count = query_includes and query_includes.__item_count or 0 --[[@as integer]]
 
@@ -4353,22 +4369,45 @@ function __evolved_execute(query)
                 (query_exclude_count == 0 or not __chunk_has_any_fragment_list(
                     major_chunk, query_exclude_list, query_exclude_count))
 
+            if is_major_chunk_matched and major_chunk.__has_hidden_minors then
+                local major_chunk_fragment_list = major_chunk.__fragment_list
+                local major_chunk_fragment_count = major_chunk.__fragment_count
+
+                for major_chunk_fragment_index = 1, major_chunk_fragment_count - 1 do
+                    local major_chunk_fragment = major_chunk_fragment_list[major_chunk_fragment_index]
+
+                    if not query_include_set[major_chunk_fragment] and __evolved_has(major_chunk_fragment, __HIDDEN) then
+                        is_major_chunk_matched = false
+                        break
+                    end
+                end
+            end
+
             if is_major_chunk_matched then
                 chunk_stack_size = chunk_stack_size + 1
                 chunk_stack[chunk_stack_size] = major_chunk
             end
         end
     elseif query_exclude_count > 0 then
         for root_fragment, root_chunk in __lua_next, __root_chunks do
-            if not query_exclude_set[root_fragment] then
+            local is_root_chunk_matched =
+                not root_chunk.__has_hidden_major and
+                not query_exclude_set[root_fragment]
+
+            if is_root_chunk_matched then
                 chunk_stack_size = chunk_stack_size + 1
                 chunk_stack[chunk_stack_size] = root_chunk
             end
         end
     else
         for _, root_chunk in __lua_next, __root_chunks do
-            chunk_stack_size = chunk_stack_size + 1
-            chunk_stack[chunk_stack_size] = root_chunk
+            local is_root_chunk_matched =
+                not root_chunk.__has_hidden_major
+
+            if is_root_chunk_matched then
+                chunk_stack_size = chunk_stack_size + 1
+                chunk_stack[chunk_stack_size] = root_chunk
+            end
         end
     end
 
@@ -5048,25 +5087,29 @@ end
 local function __update_chunk_caches_trace(chunk)
     local chunk_parent, chunk_fragment = chunk.__parent, chunk.__fragment
 
-    local has_setup_hooks = (chunk_parent and chunk_parent.__has_setup_hooks)
+    local has_setup_hooks = (chunk_parent ~= nil and chunk_parent.__has_setup_hooks)
         or __evolved_has_any(chunk_fragment, __DEFAULT, __DUPLICATE)
 
-    local has_assign_hooks = (chunk_parent and chunk_parent.__has_assign_hooks)
+    local has_assign_hooks = (chunk_parent ~= nil and chunk_parent.__has_assign_hooks)
         or __evolved_has_any(chunk_fragment, __ON_SET, __ON_ASSIGN)
 
-    local has_insert_hooks = (chunk_parent and chunk_parent.__has_insert_hooks)
+    local has_insert_hooks = (chunk_parent ~= nil and chunk_parent.__has_insert_hooks)
         or __evolved_has_any(chunk_fragment, __ON_SET, __ON_INSERT)
 
-    local has_remove_hooks = (chunk_parent and chunk_parent.__has_remove_hooks)
+    local has_remove_hooks = (chunk_parent ~= nil and chunk_parent.__has_remove_hooks)
         or __evolved_has(chunk_fragment, __ON_REMOVE)
 
-    local has_hidden_fragments = (chunk_parent and chunk_parent.__has_hidden_fragments)
-        or __evolved_has(chunk_fragment, __HIDDEN)
+    local has_hidden_major = __evolved_has(chunk_fragment, __HIDDEN)
+    local has_hidden_minors = chunk_parent ~= nil and chunk_parent.__has_hidden_fragments
+    local has_hidden_fragments = has_hidden_major or has_hidden_minors
 
     chunk.__has_setup_hooks = has_setup_hooks
     chunk.__has_assign_hooks = has_assign_hooks
     chunk.__has_insert_hooks = has_insert_hooks
     chunk.__has_remove_hooks = has_remove_hooks
+
+    chunk.__has_hidden_major = has_hidden_major
+    chunk.__has_hidden_minors = has_hidden_minors
     chunk.__has_hidden_fragments = has_hidden_fragments
 
     return true