@@ -115,32 +115,6 @@ static inline bool is_write_fault(unsigned int fsr)
115115 return (fsr & FSR_WRITE ) && !(fsr & FSR_CM );
116116}
117117
118- static inline bool is_translation_fault (unsigned int fsr )
119- {
120- int fs = fsr_fs (fsr );
121- #ifdef CONFIG_ARM_LPAE
122- if ((fs & FS_MMU_NOLL_MASK ) == FS_TRANS_NOLL )
123- return true;
124- #else
125- if (fs == FS_L1_TRANS || fs == FS_L2_TRANS )
126- return true;
127- #endif
128- return false;
129- }
130-
131- static inline bool is_permission_fault (unsigned int fsr )
132- {
133- int fs = fsr_fs (fsr );
134- #ifdef CONFIG_ARM_LPAE
135- if ((fs & FS_MMU_NOLL_MASK ) == FS_PERM_NOLL )
136- return true;
137- #else
138- if (fs == FS_L1_PERM || fs == FS_L2_PERM )
139- return true;
140- #endif
141- return false;
142- }
143-
144118static void die_kernel_fault (const char * msg , struct mm_struct * mm ,
145119 unsigned long addr , unsigned int fsr ,
146120 struct pt_regs * regs )
@@ -190,14 +164,17 @@ __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
190164
191165/*
192166 * Something tried to access memory that isn't in our memory map..
193- * User mode accesses just cause a SIGSEGV
167+ * User mode accesses just cause a SIGSEGV. Ensure interrupts are enabled
168+ * for preempt RT.
194169 */
195170static void
196171__do_user_fault (unsigned long addr , unsigned int fsr , unsigned int sig ,
197172 int code , struct pt_regs * regs )
198173{
199174 struct task_struct * tsk = current ;
200175
176+ local_irq_enable ();
177+
201178#ifdef CONFIG_DEBUG_USER
202179 if (((user_debug & UDBG_SEGV ) && (sig == SIGSEGV )) ||
203180 ((user_debug & UDBG_BUS ) && (sig == SIGBUS ))) {
@@ -258,6 +235,70 @@ static inline bool ttbr0_usermode_access_allowed(struct pt_regs *regs)
258235}
259236#endif
260237
238+ /*
239+ * Handle a vmalloc fault, copying the non-leaf page table entries from
240+ * init_mm.pgd. Any kernel context can trigger this, so we must not sleep
241+ * or enable interrupts. Having two CPUs execute this for the same page is
242+ * no problem, we'll just copy the same data twice.
243+ *
244+ * Returns false on failure.
245+ */
246+ static bool __kprobes __maybe_unused vmalloc_fault (unsigned long addr )
247+ {
248+ unsigned int index ;
249+ pgd_t * pgd , * pgd_k ;
250+ p4d_t * p4d , * p4d_k ;
251+ pud_t * pud , * pud_k ;
252+ pmd_t * pmd , * pmd_k ;
253+
254+ index = pgd_index (addr );
255+
256+ pgd = cpu_get_pgd () + index ;
257+ pgd_k = init_mm .pgd + index ;
258+
259+ p4d = p4d_offset (pgd , addr );
260+ p4d_k = p4d_offset (pgd_k , addr );
261+
262+ if (p4d_none (* p4d_k ))
263+ return false;
264+ if (!p4d_present (* p4d ))
265+ set_p4d (p4d , * p4d_k );
266+
267+ pud = pud_offset (p4d , addr );
268+ pud_k = pud_offset (p4d_k , addr );
269+
270+ if (pud_none (* pud_k ))
271+ return false;
272+ if (!pud_present (* pud ))
273+ set_pud (pud , * pud_k );
274+
275+ pmd = pmd_offset (pud , addr );
276+ pmd_k = pmd_offset (pud_k , addr );
277+
278+ #ifdef CONFIG_ARM_LPAE
279+ /*
280+ * Only one hardware entry per PMD with LPAE.
281+ */
282+ index = 0 ;
283+ #else
284+ /*
285+ * On ARM one Linux PGD entry contains two hardware entries (see page
286+ * tables layout in pgtable.h). We normally guarantee that we always
287+ * fill both L1 entries. But create_mapping() doesn't follow the rule.
288+ * It can create inidividual L1 entries, so here we have to call
289+ * pmd_none() check for the entry really corresponded to address, not
290+ * for the first of pair.
291+ */
292+ index = (addr >> SECTION_SHIFT ) & 1 ;
293+ #endif
294+ if (pmd_none (pmd_k [index ]))
295+ return false;
296+
297+ copy_pmd (pmd , pmd_k );
298+
299+ return true;
300+ }
301+
261302static int __kprobes
262303do_kernel_address_page_fault (struct mm_struct * mm , unsigned long addr ,
263304 unsigned int fsr , struct pt_regs * regs )
@@ -268,6 +309,7 @@ do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr,
268309 * should not be faulting in kernel space, which includes the
269310 * vector/khelper page. Handle the branch predictor hardening
270311 * while interrupts are still disabled, then send a SIGSEGV.
312+ * Note that __do_user_fault() will enable interrupts.
271313 */
272314 harden_branch_predictor ();
273315 __do_user_fault (addr , fsr , SIGSEGV , SEGV_MAPERR , regs );
@@ -492,10 +534,9 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
492534 * directly to do_kernel_address_page_fault() to handle.
493535 *
494536 * Otherwise, we're probably faulting in the vmalloc() area, so try to fix
495- * that up. Note that we must not take any locks or enable interrupts in
496- * this case.
537+ * that up via vmalloc_fault().
497538 *
498- * If vmalloc() fixup fails, that means the non-leaf page tables did not
539+ * If vmalloc_fault() fails, that means the non-leaf page tables did not
499540 * contain an entry for this address, so handle this via
500541 * do_kernel_address_page_fault().
501542 */
@@ -504,65 +545,12 @@ static int __kprobes
504545do_translation_fault (unsigned long addr , unsigned int fsr ,
505546 struct pt_regs * regs )
506547{
507- unsigned int index ;
508- pgd_t * pgd , * pgd_k ;
509- p4d_t * p4d , * p4d_k ;
510- pud_t * pud , * pud_k ;
511- pmd_t * pmd , * pmd_k ;
512-
513548 if (addr < TASK_SIZE )
514549 return do_page_fault (addr , fsr , regs );
515550
516- if (user_mode (regs ))
517- goto bad_area ;
518-
519- index = pgd_index (addr );
520-
521- pgd = cpu_get_pgd () + index ;
522- pgd_k = init_mm .pgd + index ;
523-
524- p4d = p4d_offset (pgd , addr );
525- p4d_k = p4d_offset (pgd_k , addr );
526-
527- if (p4d_none (* p4d_k ))
528- goto bad_area ;
529- if (!p4d_present (* p4d ))
530- set_p4d (p4d , * p4d_k );
531-
532- pud = pud_offset (p4d , addr );
533- pud_k = pud_offset (p4d_k , addr );
534-
535- if (pud_none (* pud_k ))
536- goto bad_area ;
537- if (!pud_present (* pud ))
538- set_pud (pud , * pud_k );
539-
540- pmd = pmd_offset (pud , addr );
541- pmd_k = pmd_offset (pud_k , addr );
542-
543- #ifdef CONFIG_ARM_LPAE
544- /*
545- * Only one hardware entry per PMD with LPAE.
546- */
547- index = 0 ;
548- #else
549- /*
550- * On ARM one Linux PGD entry contains two hardware entries (see page
551- * tables layout in pgtable.h). We normally guarantee that we always
552- * fill both L1 entries. But create_mapping() doesn't follow the rule.
553- * It can create inidividual L1 entries, so here we have to call
554- * pmd_none() check for the entry really corresponded to address, not
555- * for the first of pair.
556- */
557- index = (addr >> SECTION_SHIFT ) & 1 ;
558- #endif
559- if (pmd_none (pmd_k [index ]))
560- goto bad_area ;
561-
562- copy_pmd (pmd , pmd_k );
563- return 0 ;
551+ if (!user_mode (regs ) && vmalloc_fault (addr ))
552+ return 0 ;
564553
565- bad_area :
566554 do_kernel_address_page_fault (current -> mm , addr , fsr , regs );
567555
568556 return 0 ;
0 commit comments