Skip to content

Commit b6cd547

Browse files
theraulttherault
committed
Create a Priority Controlled Binding scheduler that uses some bits (MCA defined) of the priority word to define which 'group' of threads can schedule a task with this priority. See documentation in sched_pcb.h for more details
1 parent b1de9a2 commit b6cd547

11 files changed

Lines changed: 827 additions & 3 deletions

File tree

‎parsec/mca/sched/pcb/sched_pcb.h‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright (c) 2022 The University of Tennessee and The University
3+
* of Tennessee Research Foundation. All rights
4+
* reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*/
11+
12+
/**
13+
* @file
14+
*
15+
* Priority Controlled Binding scheduler
16+
*
17+
* This scheduler uses some bits of the priority word attached to each
18+
* task to define which set of threads can execute the task.
19+
*
20+
* The bits that are used in the priority are defined using the MCA
21+
* parameter sched_pcb_priority_mask, which should contain enough
22+
* consecutive bits to express a number between 0 and N (inclusive) where
23+
* N is the number of 'thread groups'.
24+
*
25+
* Each computing thread of a given process belongs to a thread group.
26+
* If PaRSEC is compiled with HWLOC, which threads belong to which group
27+
* are defined using the HWLOC tree hierarchy and the MCA parameter
28+
* sched_pcb_sharing_level: a sched_pcb_sharing_level of L means that all
29+
* threads bound to a core that is under the same node of depth L in the HWLOC
30+
* tree belong to the same group. Setting sched_pcb_sharing_level to 0 means
31+
* that all threads are in the same group (they are under the root of the tree),
32+
* and setting it to parsec_hwloc_nb_levels()-1 means that each thread is
33+
* in its own group, by itself. Intermediate values have different results
34+
* depending on the machine hierarchy.
35+
*
36+
* If PaRSEC is compiled without HWLOC, the MCA parameter is not exposed,
37+
* and there is a single behavior: each thread belongs to its own group,
38+
* by itself.
39+
*
40+
* There is a 'special' group: the group 0 (other groups are named 1 to N).
41+
* Tasks that are bound to that group are in fact shared between all threads
42+
* (as is usual for other schedulers).
43+
*
44+
* So, tasks with a priority 0 are always scheduled opportunistically on
45+
* any thread. Because startup tasks also initialize their priority to -1,
46+
* tasks with priority -1 are also handled specially by allowing any thread
47+
* to execute them (they are assigned the group 0 despite their priority).
48+
*
49+
* Last, the scheduler uses the priority value to order all the tasks that
50+
* are bound to a given group.
51+
*
52+
* At task selection time, a thread compares the priority of the highest
53+
* priority task of the group 0 and the highest priority task of its own
54+
* group, and selects the task with the highest priority.
55+
*
56+
* Access to the task lists are protected with locks.
57+
*/
58+
59+
#ifndef MCA_SCHED_PCB_H
60+
#define MCA_SCHED_PCB_H
61+
62+
#include "parsec/parsec_config.h"
63+
#include "parsec/mca/mca.h"
64+
#include "parsec/mca/sched/sched.h"
65+
66+
67+
BEGIN_C_DECLS
68+
69+
/**
70+
* Globally exported variable
71+
*/
72+
PARSEC_DECLSPEC extern const parsec_sched_base_component_t parsec_sched_pcb_component;
73+
PARSEC_DECLSPEC extern const parsec_sched_module_t parsec_sched_pcb_module;
74+
/* static accessor */
75+
mca_base_component_t *sched_pcb_static_component(void);
76+
extern int sched_pcb_sharing_level;
77+
extern int sched_pcb_group_mask;
78+
extern int sched_pcb_group_shift;
79+
80+
END_C_DECLS
81+
#endif /* MCA_SCHED_PCB_H */

‎parsec/mca/sched/pcb/sched_pcb_component.c‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright (c) 2022 The University of Tennessee and The University
3+
* of Tennessee Research Foundation. All rights
4+
* reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*
11+
* These symbols are in a file by themselves to provide nice linker
12+
* semantics. Since linkers generally pull in symbols by object
13+
* files, keeping these symbols as the only symbols in this file
14+
* prevents utility programs such as "ompi_info" from having to import
15+
* entire components just to query their version and parameters.
16+
*/
17+
18+
#include "parsec/parsec_config.h"
19+
#include "parsec/runtime.h"
20+
21+
#include "parsec/mca/sched/sched.h"
22+
#include "parsec/mca/sched/pcb/sched_pcb.h"
23+
#include "parsec/papi_sde.h"
24+
#include "parsec/utils/debug.h"
25+
#include "parsec/utils/mca_param.h"
26+
27+
#if defined(PARSEC_HAVE_HWLOC)
28+
#include "parsec/parsec_hwloc.h"
29+
#endif
30+
31+
/*
32+
* Local function
33+
*/
34+
static int sched_pcb_component_query(mca_base_module_t **module, int *priority);
35+
static int sched_pcb_component_register(void);
36+
37+
int sched_pcb_sharing_level = 1;
38+
int sched_pcb_group_mask = 0x7f000000;
39+
int sched_pcb_group_shift = 24;
40+
/*
41+
* Instantiate the public struct with all of our public information
42+
* and pointers to our public functions in it
43+
*/
44+
const parsec_sched_base_component_t parsec_sched_pcb_component = {
45+
46+
/* First, the mca_component_t struct containing meta information
47+
about the component itself */
48+
49+
{
50+
PARSEC_SCHED_BASE_VERSION_2_0_0,
51+
52+
/* Component name and version */
53+
"pcb",
54+
"", /* options */
55+
PARSEC_VERSION_MAJOR,
56+
PARSEC_VERSION_MINOR,
57+
58+
/* Component open and close functions */
59+
NULL, /*< No open: sched_pcb is always available, no need to check at runtime */
60+
NULL, /*< No close: open did not allocate any resource, no need to release them */
61+
sched_pcb_component_query,
62+
/*< specific query to return the module and add it to the list of available modules */
63+
sched_pcb_component_register, /*< Register at least the SDE events */
64+
"", /*< no reserve */
65+
},
66+
{
67+
/* The component has no metada */
68+
MCA_BASE_METADATA_PARAM_NONE,
69+
"", /*< no reserve */
70+
}
71+
};
72+
mca_base_component_t *sched_pcb_static_component(void)
73+
{
74+
return (mca_base_component_t *)&parsec_sched_pcb_component;
75+
}
76+
77+
static int sched_pcb_component_query(mca_base_module_t **module, int *priority)
78+
{
79+
/* module type should be: const mca_base_module_t ** */
80+
void *ptr = (void*)&parsec_sched_pcb_module;
81+
*priority = 2;
82+
*module = (mca_base_module_t *)ptr;
83+
return MCA_SUCCESS;
84+
}
85+
86+
static int sched_pcb_component_register(void)
87+
{
88+
PARSEC_PAPI_SDE_DESCRIBE_COUNTER("SCHEDULER::PENDING_TASKS::SCHED=PCB",
89+
"the number of pending tasks for the PCB scheduler");
90+
PARSEC_PAPI_SDE_DESCRIBE_COUNTER("SCHEDULER::PENDING_TASKS::QUEUE=<VPID>::SCHED=PCB",
91+
"the number of pending tasks that end up in the virtual process <VPID> for the LFQ scheduler");
92+
sched_pcb_sharing_level = 1;
93+
#if defined(PARSEC_HAVE_HWLOC)
94+
sched_pcb_sharing_level = parsec_hwloc_nb_levels()-1;
95+
parsec_mca_param_reg_int_name("sched_pcb", "sharing_level",
96+
"Defines at what level threads share the same task list for the Priority Controlled Binding scheduler. "
97+
"Level 1 means each thread has its own task list, level 2 looks one level above in the HWLOC hierarchy, etc...",
98+
false, false, parsec_hwloc_nb_levels()-1, &sched_pcb_sharing_level);
99+
if(sched_pcb_sharing_level <= 0)
100+
sched_pcb_sharing_level = 1;
101+
if(sched_pcb_sharing_level >= parsec_hwloc_nb_levels())
102+
sched_pcb_sharing_level = parsec_hwloc_nb_levels()-1;
103+
#endif
104+
parsec_mca_param_reg_int_name("sched_pcb", "group_mask",
105+
"Defines what bits of the priority are used to designate a process group. Other bits are of the priority value "
106+
"are used to define the priority of the task within that group.",
107+
false, false, 0x7f000000, &sched_pcb_group_mask);
108+
if(sched_pcb_group_mask != 0x7f000000) {
109+
sched_pcb_group_shift = 0;
110+
while( (unsigned int)sched_pcb_group_shift < 8*sizeof(int) &&
111+
(((sched_pcb_group_mask >> sched_pcb_group_shift) & 1) == 0) )
112+
sched_pcb_group_shift++;
113+
if(sched_pcb_group_shift == 8*sizeof(int)) {
114+
parsec_warning("Priority Controlled Binding Scheduler (sched_pcb): sched_pcb_group_mask is set to 0. Scheduler might not work as intended.");
115+
}
116+
}
117+
118+
return MCA_SUCCESS;
119+
}

0 commit comments

Comments
 (0)