From: David Gibson This has been given basic testing on Power4 pSeries and iSeries machines. At present, the SLB miss handler has to check the SLB slot it is about to use to ensure that it does not contain the SLBE for the current kernel stack - throwing out the SLBE for the kernel stack can trigger the "megabug": we have no SLBE for the stack, but we don't fault immediately because we have an ERAT entry for it. The ERAT entry is then lost due to a tlbie on another CPU during the unrecoverable section of the exception return path. This patch implements a different approach - with this patch SLB slot 2 always (well, nearly always) contains an SLBE for the stack. This slot is never cast out by the normal SLB miss path. On context switch, an SLBE for the new stack is pinned into this slot, unless the new stack is the the bolted segment. For iSeries we need a special workaround because there is no way of ensuring the stack SLBE is preserved an a shared processor switch. So, we still need to handle taking an SLB miss on the stack, in which case we must make sure it is loaded into slot 2, rather than using the normal round-robin. This approach shaves a few ns off the slb miss time (on pSeries), but more importantly makes it easier to experiment with different SLB castout aporoaches without worrying about reinstating the megabug. Signed-off-by: David Gibson Signed-off-by: Andrew Morton --- 25-akpm/arch/ppc64/kernel/asm-offsets.c | 1 25-akpm/arch/ppc64/kernel/entry.S | 14 ++++- 25-akpm/arch/ppc64/kernel/process.c | 10 +++ 25-akpm/arch/ppc64/mm/slb.c | 85 +++++++++++++++++++++----------- 25-akpm/arch/ppc64/mm/slb_low.S | 46 ++++++----------- 25-akpm/include/asm-ppc64/mmu.h | 2 25-akpm/include/asm-ppc64/processor.h | 1 7 files changed, 97 insertions(+), 62 deletions(-) diff -puN arch/ppc64/kernel/asm-offsets.c~pin-the-kernel-stacks-slb-entry arch/ppc64/kernel/asm-offsets.c --- 25/arch/ppc64/kernel/asm-offsets.c~pin-the-kernel-stacks-slb-entry 2004-08-31 21:28:35.262641272 -0700 +++ 25-akpm/arch/ppc64/kernel/asm-offsets.c 2004-08-31 21:28:35.275639296 -0700 @@ -58,6 +58,7 @@ int main(void) DEFINE(THREAD_FPR0, offsetof(struct thread_struct, fpr[0])); DEFINE(THREAD_FPSCR, offsetof(struct thread_struct, fpscr)); DEFINE(KSP, offsetof(struct thread_struct, ksp)); + DEFINE(KSP_VSID, offsetof(struct thread_struct, ksp_vsid)); #ifdef CONFIG_ALTIVEC DEFINE(THREAD_VR0, offsetof(struct thread_struct, vr[0])); diff -puN arch/ppc64/kernel/entry.S~pin-the-kernel-stacks-slb-entry arch/ppc64/kernel/entry.S --- 25/arch/ppc64/kernel/entry.S~pin-the-kernel-stacks-slb-entry 2004-08-31 21:28:35.263641120 -0700 +++ 25-akpm/arch/ppc64/kernel/entry.S 2004-08-31 21:28:35.276639144 -0700 @@ -393,9 +393,17 @@ BEGIN_FTR_SECTION cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */ cror eq,4*cr1+eq,eq beq 2f /* if yes, don't slbie it */ - oris r6,r6,0x0800 /* set C (class) bit */ - slbie r6 - slbie r6 /* Workaround POWER5 < DD2.1 issue */ + oris r0,r6,0x0800 /* set C (class) bit */ + + /* Bolt in the new stack SLB entry */ + ld r7,KSP_VSID(r4) /* Get new stack's VSID */ + oris r6,r6,(SLB_ESID_V)@h + ori r6,r6,(SLB_NUM_BOLTED-1)@l + slbie r0 + slbie r0 /* Workaround POWER5 < DD2.1 issue */ + slbmte r7,r6 + isync + 2: END_FTR_SECTION_IFSET(CPU_FTR_SLB) clrrdi r7,r8,THREAD_SHIFT /* base of new stack */ diff -puN arch/ppc64/kernel/process.c~pin-the-kernel-stacks-slb-entry arch/ppc64/kernel/process.c --- 25/arch/ppc64/kernel/process.c~pin-the-kernel-stacks-slb-entry 2004-08-31 21:28:35.265640816 -0700 +++ 25-akpm/arch/ppc64/kernel/process.c 2004-08-31 21:28:35.276639144 -0700 @@ -356,6 +356,16 @@ copy_thread(int nr, unsigned long clone_ kregs = (struct pt_regs *) sp; sp -= STACK_FRAME_OVERHEAD; p->thread.ksp = sp; + if (cur_cpu_spec->cpu_features & CPU_FTR_SLB) { + unsigned long sp_vsid = get_kernel_vsid(sp); + + sp_vsid <<= SLB_VSID_SHIFT; + sp_vsid |= SLB_VSID_KERNEL; + if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) + sp_vsid |= SLB_VSID_L; + + p->thread.ksp_vsid = sp_vsid; + } /* * The PPC64 ABI makes use of a TOC to contain function diff -puN arch/ppc64/mm/slb.c~pin-the-kernel-stacks-slb-entry arch/ppc64/mm/slb.c --- 25/arch/ppc64/mm/slb.c~pin-the-kernel-stacks-slb-entry 2004-08-31 21:28:35.267640512 -0700 +++ 25-akpm/arch/ppc64/mm/slb.c 2004-08-31 21:28:35.277638992 -0700 @@ -24,30 +24,55 @@ extern void slb_allocate(unsigned long ea); +static inline unsigned long mk_esid_data(unsigned long ea, unsigned long slot) +{ + return (ea & ESID_MASK) | SLB_ESID_V | slot; +} + +static inline unsigned long mk_vsid_data(unsigned long ea, unsigned long flags) +{ + return (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | flags; +} + static inline void create_slbe(unsigned long ea, unsigned long vsid, unsigned long flags, unsigned long entry) { - ea = (ea & ESID_MASK) | SLB_ESID_V | entry; - vsid = (vsid << SLB_VSID_SHIFT) | flags; asm volatile("slbmte %0,%1" : - : "r" (vsid), "r" (ea) + : "r" (mk_vsid_data(ea, flags)), + "r" (mk_esid_data(ea, entry)) : "memory" ); } -static void slb_add_bolted(void) +static void slb_flush_and_rebolt(void) { - WARN_ON(!irqs_disabled()); - /* If you change this make sure you change SLB_NUM_BOLTED - * appropriately too */ + * appropriately too. */ + unsigned long ksp_flags = SLB_VSID_KERNEL; + unsigned long ksp_esid_data; - /* Slot 1 - first VMALLOC segment - * Since modules end up there it gets hit very heavily. - */ - create_slbe(VMALLOCBASE, get_kernel_vsid(VMALLOCBASE), - SLB_VSID_KERNEL, 1); + WARN_ON(!irqs_disabled()); - asm volatile("isync":::"memory"); + if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) + ksp_flags |= SLB_VSID_L; + + ksp_esid_data = mk_esid_data(get_paca()->kstack, 2); + if ((ksp_esid_data & ESID_MASK) == KERNELBASE) + ksp_esid_data &= ~SLB_ESID_V; + + /* We need to do this all in asm, so we're sure we don't touch + * the stack between the slbia and rebolting it. */ + asm volatile("isync\n" + "slbia\n" + /* Slot 1 - first VMALLOC segment */ + "slbmte %0,%1\n" + /* Slot 2 - kernel stack */ + "slbmte %2,%3\n" + "isync" + :: "r"(mk_vsid_data(VMALLOCBASE, SLB_VSID_KERNEL)), + "r"(mk_esid_data(VMALLOCBASE, 1)), + "r"(mk_vsid_data(ksp_esid_data, ksp_flags)), + "r"(ksp_esid_data) + : "memory"); } /* Flush all user entries from the segment table of the current processor. */ @@ -69,8 +94,7 @@ void switch_slb(struct task_struct *tsk, } asm volatile("isync" : : : "memory"); } else { - asm volatile("isync; slbia; isync" : : : "memory"); - slb_add_bolted(); + slb_flush_and_rebolt(); } /* Workaround POWER5 < DD2.1 issue */ @@ -113,22 +137,27 @@ void switch_slb(struct task_struct *tsk, void slb_initialize(void) { -#ifdef CONFIG_PPC_ISERIES - asm volatile("isync; slbia; isync":::"memory"); -#else + /* On iSeries the bolted entries have already been set up by + * the hypervisor from the lparMap data in head.S */ +#ifndef CONFIG_PPC_ISERIES unsigned long flags = SLB_VSID_KERNEL; - /* Invalidate the entire SLB (even slot 0) & all the ERATS */ - if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) - flags |= SLB_VSID_L; - - asm volatile("isync":::"memory"); - asm volatile("slbmte %0,%0"::"r" (0) : "memory"); + /* Invalidate the entire SLB (even slot 0) & all the ERATS */ + if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) + flags |= SLB_VSID_L; + + asm volatile("isync":::"memory"); + asm volatile("slbmte %0,%0"::"r" (0) : "memory"); asm volatile("isync; slbia; isync":::"memory"); - create_slbe(KERNELBASE, get_kernel_vsid(KERNELBASE), - flags, 0); - + create_slbe(KERNELBASE, get_kernel_vsid(KERNELBASE), flags, 0); + create_slbe(VMALLOCBASE, get_kernel_vsid(KERNELBASE), + SLB_VSID_KERNEL, 1); + /* We don't bolt the stack for the time being - we're in boot, + * so the stack is in the bolted segment. By the time it goes + * elsewhere, we'll call _switch() which will bolt in the new + * one. */ + asm volatile("isync":::"memory"); #endif - slb_add_bolted(); + get_paca()->stab_rr = SLB_NUM_BOLTED; } diff -puN arch/ppc64/mm/slb_low.S~pin-the-kernel-stacks-slb-entry arch/ppc64/mm/slb_low.S --- 25/arch/ppc64/mm/slb_low.S~pin-the-kernel-stacks-slb-entry 2004-08-31 21:28:35.268640360 -0700 +++ 25-akpm/arch/ppc64/mm/slb_low.S 2004-08-31 21:28:35.278638840 -0700 @@ -37,8 +37,21 @@ _GLOBAL(slb_allocate) * a free slot first but that took too long. Unfortunately we * dont have any LRU information to help us choose a slot. */ +#ifdef CONFIG_PPC_ISERIES + /* + * On iSeries, the "bolted" stack segment can be cast out on + * shared processor switch so we need to check for a miss on + * it and restore it to the right slot. + */ + ld r9,PACAKSAVE(r13) + clrrdi r9,r9,28 + clrrdi r11,r3,28 + li r10,SLB_NUM_BOLTED-1 /* Stack goes in last bolted slot */ + cmpld r9,r11 + beq 3f +#endif /* CONFIG_PPC_ISERIES */ + ld r10,PACASTABRR(r13) -3: addi r10,r10,1 /* use a cpu feature mask if we ever change our slb size */ cmpldi r10,SLB_NUM_ENTRIES @@ -46,36 +59,9 @@ _GLOBAL(slb_allocate) blt+ 4f li r10,SLB_NUM_BOLTED - /* - * Never cast out the segment for our kernel stack. Since we - * dont invalidate the ERAT we could have a valid translation - * for the kernel stack during the first part of exception exit - * which gets invalidated due to a tlbie from another cpu at a - * non recoverable point (after setting srr0/1) - Anton - */ -4: slbmfee r11,r10 - srdi r11,r11,27 - /* - * Use paca->ksave as the value of the kernel stack pointer, - * because this is valid at all times. - * The >> 27 (rather than >> 28) is so that the LSB is the - * valid bit - this way we check valid and ESID in one compare. - * In order to completely close the tiny race in the context - * switch (between updating r1 and updating paca->ksave), - * we check against both r1 and paca->ksave. - */ - srdi r9,r1,27 - ori r9,r9,1 /* mangle SP for later compare */ - cmpd r11,r9 - beq- 3b - ld r9,PACAKSAVE(r13) - srdi r9,r9,27 - ori r9,r9,1 - cmpd r11,r9 - beq- 3b - +4: std r10,PACASTABRR(r13) - +3: /* r3 = faulting address, r10 = entry */ srdi r9,r3,60 /* get region */ diff -puN include/asm-ppc64/mmu.h~pin-the-kernel-stacks-slb-entry include/asm-ppc64/mmu.h --- 25/include/asm-ppc64/mmu.h~pin-the-kernel-stacks-slb-entry 2004-08-31 21:28:35.270640056 -0700 +++ 25-akpm/include/asm-ppc64/mmu.h 2004-08-31 21:28:35.278638840 -0700 @@ -198,7 +198,7 @@ extern void htab_finish_init(void); #define STAB0_PHYS_ADDR (STAB0_PAGE<