Measure and report executor starvation time

The executor is considered to be "starving" when it is out of
instructions to process, and the scheduler is currently busy.
This means that phases of the user program that do not interact with the
Celerity API do not count as starvation periods.

If the total starvation time exceeds a percentage threshold of the time
spent doing actual work (i.e., processing instructions), we print a
warning indicating that the application might be scheduler-bound.

Author: psalz

include/double_buffered_queue.h

@@ -39,6 +39,9 @@ class double_buffered_queue {
 		return m_read.queue;
 	}
 
+	/// Returns true if a call to `pop_all` would have returned an empty vector.
+	bool empty() const { return !(m_write.queue_nonempty.load(std::memory_order_relaxed)); }
+
 	/// After this function returns, the result of `pop_all` is non-empty as long as there is only a single reader thread.
 	void wait_while_empty() {
 		if(!m_write.queue_nonempty.load(std::memory_order_relaxed) /* opportunistic */) {

include/dry_run_executor.h

@@ -35,6 +35,11 @@ class dry_run_executor final : public executor {
 
 	void submit(std::vector<const instruction*> instructions, std::vector<outbound_pilot> pilots) override;
 
+	void notify_scheduler_idle(bool is_idle) override;
+
+	std::chrono::nanoseconds get_starvation_time() const override;
+	std::chrono::nanoseconds get_active_time() const override;
+
   private:
 	using host_object_transfer = std::pair<host_object_id, std::unique_ptr<host_object_instance>>;
 	using submission = std::variant<std::vector<const instruction*>, host_object_transfer>;

include/executor.h

@@ -2,6 +2,7 @@
 
 #include "types.h"
 
+#include <chrono>
 #include <memory>
 #include <vector>
 
@@ -61,6 +62,15 @@ class executor {
 	/// recipients as soon as possible. Instructions must be in topological order of dependencies, as must be the concatenation of all vectors passed to
 	/// subsequent invocations of this function.
 	virtual void submit(std::vector<const instruction*> instructions, std::vector<outbound_pilot> pilots) = 0;
+
+	/// Informs the executor about a change of the scheduler idle state. Required for tracking starvation time.
+	virtual void notify_scheduler_idle(const bool is_idle) = 0;
+
+	/// Returns the total time the executor has spent idle waiting for instructions while the scheduler was busy.
+	virtual std::chrono::nanoseconds get_starvation_time() const = 0;
+
+	/// Returns the total time the executor has spent processing instructions.
+	virtual std::chrono::nanoseconds get_active_time() const = 0;
 };
 
 } // namespace celerity::detail

include/live_executor.h

@@ -30,14 +30,16 @@ struct reducer_transfer {
 	reduction_id rid = 0;
 	std::unique_ptr<reducer> reduction;
 };
-using submission = std::variant<instruction_pilot_batch, user_allocation_transfer, host_object_transfer, reducer_transfer>;
+struct scheduler_idle_state_change {
+	bool is_idle = false;
+};
+using submission = std::variant<instruction_pilot_batch, user_allocation_transfer, host_object_transfer, reducer_transfer, scheduler_idle_state_change>;
 
 } // namespace celerity::detail::live_executor_detail
 
 namespace celerity::detail {
 
 class communicator;
-struct system_info;
 class backend;
 
 /// Executor implementation for a normal (non-dry) run of a Celerity application. Internal instruction dependencies are resolved by means of an
@@ -66,14 +68,23 @@ class live_executor final : public executor {
 
 	void submit(std::vector<const instruction*> instructions, std::vector<outbound_pilot> pilots) override;
 
+	void notify_scheduler_idle(const bool is_idle) override;
+
+	std::chrono::nanoseconds get_starvation_time() const override;
+
+	std::chrono::nanoseconds get_active_time() const override;
+
   private:
 	friend struct executor_testspy;
 
+	struct impl;
+
 	std::unique_ptr<communicator> m_root_comm; // created and destroyed outside of executor thread
 	double_buffered_queue<live_executor_detail::submission> m_submission_queue;
+	std::unique_ptr<impl> m_impl;
 	std::thread m_thread;
 
-	void thread_main(std::unique_ptr<backend> backend, executor::delegate* dlg, const policy_set& policy);
+	void thread_main();
 
 	/// Default-constructs a `policy_set` - this must be a function because we can't use the implicit default constructor of `policy_set`, which has member
 	/// initializers, within its surrounding class (Clang diagnostic).

include/scheduler.h

@@ -28,7 +28,19 @@ class scheduler {
 	friend struct scheduler_testspy;
 
   public:
-	using delegate = instruction_graph_generator::delegate;
+	class delegate : public instruction_graph_generator::delegate {
+	  protected:
+		delegate() = default;
+		delegate(const delegate&) = default;
+		delegate(delegate&&) = default;
+		delegate& operator=(const delegate&) = default;
+		delegate& operator=(delegate&&) = default;
+		~delegate() = default; // do not allow destruction through base pointer
+
+	  public:
+		virtual void on_scheduler_idle() = 0;
+		virtual void on_scheduler_busy() = 0;
+	};
 
 	struct policy_set {
 		detail::command_graph_generator::policy_set command_graph_generator;

include/tracy.h

@@ -110,6 +110,7 @@ enum class trace_color : std::underlying_type_t<tracy::Color::ColorType> {
 	distr_queue_submit = tracy::Color::Orange3,
 
 	executor_fetch = tracy::Color::Gray,
+	executor_idle = tracy::Color::SlateGray,
 	executor_issue = tracy::Color::Blue,
 	executor_issue_copy = tracy::Color::Green4,
 	executor_issue_device_kernel = tracy::Color::Yellow2,

src/dry_run_executor.cc

@@ -31,6 +31,14 @@ void dry_run_executor::submit(std::vector<const instruction*> instructions, std:
 	(void)pilots; // ignore;
 }
 
+void dry_run_executor::notify_scheduler_idle(const bool is_idle) {
+	(void)is_idle; // ignore
+}
+
+std::chrono::nanoseconds dry_run_executor::get_starvation_time() const { return std::chrono::nanoseconds(0); }
+
+std::chrono::nanoseconds dry_run_executor::get_active_time() const { return std::chrono::nanoseconds(0); }
+
 void dry_run_executor::thread_main(executor::delegate* const dlg) {
 	name_and_pin_and_order_this_thread(named_threads::thread_type::executor);
 	// For simplicity we keep all executor state within this function.