Skip to content

Commit ed4e0b8

Browse files
nickdriscoll-lunargnickdriscoll-lunarg
committed
Track fences per-device and wait/signal them per loop iteration as
needed
1 parent 278bc05 commit ed4e0b8

2 files changed

Lines changed: 223 additions & 9 deletions

File tree

‎framework/decode/vulkan_replay_frame_loop_consumer.cpp‎

Lines changed: 196 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,20 @@
2727
GFXRECON_BEGIN_NAMESPACE(gfxrecon)
2828
GFXRECON_BEGIN_NAMESPACE(decode)
2929

30+
#define CHECK_VK_RESULT(_res_, _func_) \
31+
{ \
32+
VkResult _RES = (_res_); \
33+
if (_RES != VK_SUCCESS) \
34+
{ \
35+
GFXRECON_LOG_ERROR("[%s:%u] synthetic call to %s failed with %s", \
36+
__FILE__, \
37+
__LINE__, \
38+
_func_, \
39+
util::ToString(_RES).c_str()); \
40+
std::exit(-1); \
41+
} \
42+
}
43+
3044
void VulkanReplayFrameLoopConsumer::Process_vkCreateInstance(
3145
const ApiCallInfo& call_info,
3246
VkResult returnValue,
@@ -319,21 +333,195 @@ void VulkanReplayFrameLoopConsumer::Process_vkAllocateDescriptorSets(
319333
call_info, returnValue, device, pAllocateInfo, pDescriptorSets);
320334
}
321335

336+
void VulkanReplayFrameLoopConsumer::Process_vkWaitForFences(const ApiCallInfo& call_info,
337+
VkResult returnValue,
338+
format::HandleId device,
339+
uint32_t fenceCount,
340+
HandlePointerDecoder<VkFence>* pFences,
341+
VkBool32 waitAll,
342+
uint64_t timeout)
343+
{
344+
if (frame_loop_info_.IsLooping() && !frame_loop_info_.IsRepetition())
345+
{
346+
for (int i = 0; i < fenceCount; ++i)
347+
{
348+
if (!per_device_fence_tracking_.contains(device))
349+
{
350+
per_device_fence_tracking_[device] = {};
351+
}
352+
FenceTracking& t = per_device_fence_tracking_[device];
353+
354+
format::HandleId fence = pFences->GetPointer()[i];
355+
if (t.waited_upon_fences_.contains(fence))
356+
{
357+
t.waited_upon_fences_[fence] += 1;
358+
}
359+
else
360+
{
361+
t.waited_upon_fences_[fence] = 1;
362+
}
363+
GFXRECON_LOG_DEBUG("VkFence with handle \"%" PRIu64 "\" has been waited on %" PRIu32 " times.",
364+
fence,
365+
t.waited_upon_fences_[fence]);
366+
}
367+
}
368+
369+
VulkanReplayConsumer::Process_vkWaitForFences(
370+
call_info, returnValue, device, fenceCount, pFences, waitAll, timeout);
371+
}
372+
373+
void VulkanReplayFrameLoopConsumer::Process_vkQueueSubmit(const ApiCallInfo& call_info,
374+
VkResult returnValue,
375+
format::HandleId queue,
376+
uint32_t submitCount,
377+
StructPointerDecoder<Decoded_VkSubmitInfo>* pSubmits,
378+
format::HandleId fence)
379+
{
380+
if (frame_loop_info_.IsLooping() && !frame_loop_info_.IsRepetition())
381+
{
382+
// Collect fences submitted during the looping frame
383+
VulkanFenceInfo* fence_info = GetObjectInfoTable().GetVkFenceInfo(fence);
384+
if (fence_info != nullptr)
385+
{
386+
format::HandleId device = GetObjectInfoTable().GetVkQueueInfo(queue)->parent_id;
387+
if (!per_device_fence_tracking_.contains(device))
388+
{
389+
per_device_fence_tracking_[device] = {};
390+
}
391+
FenceTracking& t = per_device_fence_tracking_[device];
392+
393+
if (t.signaled_fences_.contains(fence))
394+
{
395+
t.signaled_fences_[fence] += 1;
396+
}
397+
else
398+
{
399+
t.signaled_fences_[fence] = 1;
400+
}
401+
t.signaled_fences_[fence];
402+
GFXRECON_LOG_DEBUG("VkFence with handle \"%" PRIu64 "\" has been signaled %" PRIu32 " times.",
403+
fence,
404+
t.signaled_fences_[fence]);
405+
}
406+
}
407+
408+
VulkanReplayConsumer::Process_vkQueueSubmit(call_info, returnValue, queue, submitCount, pSubmits, fence);
409+
}
410+
411+
void VulkanReplayFrameLoopConsumer::FixupDeviceFences(format::HandleId device, format::HandleId queue)
412+
{
413+
// Get fence tracking info associated with this VkDevice
414+
if (!per_device_fence_tracking_.contains(device))
415+
{
416+
// No need to fixup fences if there weren't any
417+
return;
418+
}
419+
FenceTracking& t = per_device_fence_tracking_[device];
420+
421+
VulkanObjectInfoTable& table = GetObjectInfoTable();
422+
VkDevice vk_device = table.GetVkDeviceInfo(device)->handle;
423+
const graphics::VulkanDeviceTable* device_table = GetDeviceTable(vk_device);
424+
425+
// Gather fences that need to be synthetically waited on
426+
std::vector<VkFence> manual_wait_fences;
427+
manual_wait_fences.reserve(t.signaled_fences_.size());
428+
for (auto [fence_id, signal_count] : t.signaled_fences_)
429+
{
430+
uint32_t wait_count = 0;
431+
if (t.waited_upon_fences_.contains(fence_id))
432+
{
433+
wait_count = t.waited_upon_fences_[fence_id];
434+
}
435+
436+
// Manually wait on the fence if it's signaled more times than it is waited upon
437+
if (signal_count > wait_count)
438+
{
439+
GFXRECON_LOG_DEBUG("Will synthetically wait on fence %" PRIu64, fence_id);
440+
VulkanFenceInfo* fence_info = table.GetVkFenceInfo(fence_id);
441+
manual_wait_fences.push_back(fence_info->handle);
442+
}
443+
}
444+
445+
// Gather fences that need to be synthetically signaled
446+
std::vector<VkFence> manual_signal_fences;
447+
manual_signal_fences.reserve(t.signaled_fences_.size());
448+
for (auto [fence_id, wait_count] : t.waited_upon_fences_)
449+
{
450+
uint32_t signal_count = 0;
451+
if (t.signaled_fences_.contains(fence_id))
452+
{
453+
signal_count = t.signaled_fences_[fence_id];
454+
}
455+
456+
// Manually signal fence if it's waited on more times than it is signaled
457+
if (wait_count > signal_count)
458+
{
459+
GFXRECON_LOG_DEBUG("Will synthetically signal fence %" PRIu64, fence_id);
460+
VulkanFenceInfo* fence_info = table.GetVkFenceInfo(fence_id);
461+
manual_signal_fences.push_back(fence_info->handle);
462+
}
463+
}
464+
465+
VkResult result;
466+
467+
if (manual_wait_fences.size() > 0)
468+
{
469+
GFXRECON_LOG_DEBUG("Synthetically waiting on fences...");
470+
result = device_table->WaitForFences(vk_device,
471+
manual_wait_fences.size(),
472+
manual_wait_fences.data(),
473+
VK_TRUE,
474+
std::numeric_limits<uint64_t>::max());
475+
CHECK_VK_RESULT(result, "vkWaitForFences");
476+
477+
GFXRECON_LOG_DEBUG("Resetting synthetically waited on fences...");
478+
result = device_table->ResetFences(vk_device, manual_wait_fences.size(), manual_wait_fences.data());
479+
CHECK_VK_RESULT(result, "vkResetFences");
480+
}
481+
482+
if (manual_signal_fences.size() > 0)
483+
{
484+
// Fences may have been waited on but not reset, so we reset the fences we're going
485+
// to manually signal here just in case.
486+
result = device_table->ResetFences(vk_device, manual_signal_fences.size(), manual_signal_fences.data());
487+
CHECK_VK_RESULT(result, "vkResetFences");
488+
489+
GFXRECON_LOG_DEBUG("Synthetically signaling fences...");
490+
for (VkFence fence : manual_signal_fences)
491+
{
492+
VulkanQueueInfo* queue_info = table.GetVkQueueInfo(queue);
493+
result = device_table->QueueSubmit(queue_info->handle, 0, nullptr, fence);
494+
CHECK_VK_RESULT(result, "vkDeviceWaitIdle");
495+
}
496+
}
497+
}
498+
322499
void VulkanReplayFrameLoopConsumer::Process_vkQueuePresentKHR(
323500
const ApiCallInfo& call_info,
324501
VkResult returnValue,
325502
format::HandleId queue,
326503
StructPointerDecoder<Decoded_VkPresentInfoKHR>* pPresentInfo)
327504
{
328-
// Get device
329-
CommonObjectInfoTable& table = GetObjectInfoTable();
330-
VulkanQueueInfo* queue_info = table.GetVkQueueInfo(queue);
331-
VkDevice device = queue_info->parent;
332-
GFXRECON_ASSERT(device);
333-
const graphics::VulkanDeviceTable* device_table = GetDeviceTable(device);
334-
GFXRECON_ASSERT(device_table);
335-
336505
VulkanReplayConsumer::Process_vkQueuePresentKHR(call_info, returnValue, queue, pPresentInfo);
506+
507+
if (frame_loop_info_.IsLooping())
508+
{
509+
// Get device
510+
CommonObjectInfoTable& table = GetObjectInfoTable();
511+
VulkanQueueInfo* queue_info = table.GetVkQueueInfo(queue);
512+
VkDevice device = queue_info->parent;
513+
GFXRECON_ASSERT(device);
514+
const graphics::VulkanDeviceTable* device_table = GetDeviceTable(device);
515+
GFXRECON_ASSERT(device_table);
516+
517+
VkResult result;
518+
519+
GFXRECON_LOG_DEBUG("Waiting for device to idle...");
520+
result = device_table->DeviceWaitIdle(device);
521+
CHECK_VK_RESULT(result, "vkDeviceWaitIdle");
522+
523+
FixupDeviceFences(queue_info->parent_id, queue);
524+
}
337525
}
338526

339527
GFXRECON_END_NAMESPACE(decode)

‎framework/decode/vulkan_replay_frame_loop_consumer.h‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,13 +154,39 @@ class VulkanReplayFrameLoopConsumer : public VulkanReplayConsumer
154154
StructPointerDecoder<Decoded_VkDescriptorSetAllocateInfo>* pAllocateInfo,
155155
HandlePointerDecoder<VkDescriptorSet>* pDescriptorSets) override;
156156

157+
void Process_vkWaitForFences(const ApiCallInfo& call_info,
158+
VkResult returnValue,
159+
format::HandleId device,
160+
uint32_t fenceCount,
161+
HandlePointerDecoder<VkFence>* pFences,
162+
VkBool32 waitAll,
163+
uint64_t timeout) override;
164+
165+
void Process_vkQueueSubmit(const ApiCallInfo& call_info,
166+
VkResult returnValue,
167+
format::HandleId queue,
168+
uint32_t submitCount,
169+
StructPointerDecoder<Decoded_VkSubmitInfo>* pSubmits,
170+
format::HandleId fence) override;
171+
157172
void Process_vkQueuePresentKHR(const ApiCallInfo& call_info,
158173
VkResult returnValue,
159174
format::HandleId queue,
160175
StructPointerDecoder<Decoded_VkPresentInfoKHR>* pPresentInfo) override;
161176

177+
// Private declarations
178+
private:
179+
struct FenceTracking
180+
{
181+
std::unordered_map<format::HandleId, uint32_t> signaled_fences_;
182+
std::unordered_map<format::HandleId, uint32_t> waited_upon_fences_;
183+
};
184+
void FixupDeviceFences(format::HandleId device, format::HandleId queue);
185+
186+
// Private data
162187
private:
163-
graphics::FrameLoopInfo& frame_loop_info_;
188+
graphics::FrameLoopInfo& frame_loop_info_;
189+
std::unordered_map<format::HandleId, FenceTracking> per_device_fence_tracking_;
164190
};
165191

166192
GFXRECON_END_NAMESPACE(decode)

0 commit comments

Comments
 (0)