From: Zwane Mwaikambo Signed-off-by: Zwane Mwaikambo Signed-off-by: Andrew Morton --- 25-akpm/arch/i386/kernel/time.c | 13 ++++++++ 25-akpm/arch/i386/kernel/vmlinux.lds.S | 1 25-akpm/arch/i386/lib/Makefile | 1 25-akpm/arch/i386/lib/spinlock.c | 42 +++++++++++++++++++++++++++ 25-akpm/arch/i386/oprofile/op_model_athlon.c | 2 - 25-akpm/arch/i386/oprofile/op_model_p4.c | 2 - 25-akpm/arch/i386/oprofile/op_model_ppro.c | 2 - 25-akpm/include/asm-i386/ptrace.h | 4 ++ 25-akpm/include/asm-i386/spinlock.h | 42 ++++----------------------- 9 files changed, 71 insertions(+), 38 deletions(-) diff -puN arch/i386/kernel/time.c~completely-out-of-line-spinlocks--i386 arch/i386/kernel/time.c --- 25/arch/i386/kernel/time.c~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.584557016 -0700 +++ 25-akpm/arch/i386/kernel/time.c 2004-08-30 00:20:05.603554128 -0700 @@ -200,6 +200,19 @@ unsigned long long monotonic_clock(void) } EXPORT_SYMBOL(monotonic_clock); +#ifdef CONFIG_SMP +unsigned long profile_pc(struct pt_regs *regs) +{ + unsigned long pc = instruction_pointer(regs); + + if (pc >= (unsigned long)&__spinlock_text_start && + pc <= (unsigned long)&__spinlock_text_end) + return *(unsigned long *)regs->esp; + + return pc; +} +EXPORT_SYMBOL(profile_pc); +#endif /* * timer_interrupt() needs to keep up the real-time clock, diff -puN arch/i386/kernel/vmlinux.lds.S~completely-out-of-line-spinlocks--i386 arch/i386/kernel/vmlinux.lds.S --- 25/arch/i386/kernel/vmlinux.lds.S~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.585556864 -0700 +++ 25-akpm/arch/i386/kernel/vmlinux.lds.S 2004-08-30 00:20:05.603554128 -0700 @@ -18,6 +18,7 @@ SECTIONS .text : { *(.text) SCHED_TEXT + SPINLOCK_TEXT *(.fixup) *(.gnu.warning) } = 0x9090 diff -puN arch/i386/lib/Makefile~completely-out-of-line-spinlocks--i386 arch/i386/lib/Makefile --- 25/arch/i386/lib/Makefile~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.587556560 -0700 +++ 25-akpm/arch/i386/lib/Makefile 2004-08-30 00:20:05.604553976 -0700 @@ -6,6 +6,7 @@ lib-y = checksum.o delay.o usercopy.o getuser.o memcpy.o strstr.o \ bitops.o +lib-$(CONFIG_SMP) += spinlock.o lib-$(CONFIG_X86_USE_3DNOW) += mmx.o lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o lib-$(CONFIG_KGDB) += kgdb_serial.o diff -puN /dev/null arch/i386/lib/spinlock.c --- /dev/null 2003-09-15 06:40:47.000000000 -0700 +++ 25-akpm/arch/i386/lib/spinlock.c 2004-08-30 00:20:05.604553976 -0700 @@ -0,0 +1,42 @@ +/* + * arch/i386/lib/spinlock.c + * Copyright (C) 2004 Linus Torvalds + * + * Author: Zwane Mwaikambo + */ +#include +#include + +#define __lockfunc fastcall __attribute__((section(".spinlock.text"))) +void __lockfunc __spin_lock_loop_flags(spinlock_t *lock, unsigned long flags) +{ + __asm__ __volatile__ ( "1: lock; decb (%0)\n\t" + "js 2f\n\t" + "jmp 4f\n\t" + "2: testl $0x200, %1\n\t" + "jz 3f\n\t" + "sti\n\t" + "3: rep; nop\n\t" + "cmpb $0, (%0)\n\t" + "jle 3b\n\t" + "cli\n\t" + "jmp 1b\n\t" + "4: nop\n\t" + : : "r"(&lock->lock), "r"(flags) : "memory"); +} +EXPORT_SYMBOL(__spin_lock_loop_flags); + +void __lockfunc __spin_lock_loop(spinlock_t *lock) +{ + __asm__ __volatile__ ( "1: lock; decb (%0)\n\t" + "js 2f\n\t" + "jmp 3f\n\t" + "2: rep; nop\n\t" + "cmpb $0, (%0)\n\t" + "jle 2b\n\t" + "jmp 1b\n\t" + "3: nop\n\t" + : : "r"(&lock->lock) : "memory"); +} +EXPORT_SYMBOL(__spin_lock_loop); + diff -puN arch/i386/oprofile/op_model_athlon.c~completely-out-of-line-spinlocks--i386 arch/i386/oprofile/op_model_athlon.c --- 25/arch/i386/oprofile/op_model_athlon.c~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.590556104 -0700 +++ 25-akpm/arch/i386/oprofile/op_model_athlon.c 2004-08-30 00:20:05.604553976 -0700 @@ -96,7 +96,7 @@ static int athlon_check_ctrs(unsigned in { unsigned int low, high; int i; - unsigned long eip = instruction_pointer(regs); + unsigned long eip = profile_pc(regs); int is_kernel = !user_mode(regs); for (i = 0 ; i < NUM_COUNTERS; ++i) { diff -puN arch/i386/oprofile/op_model_p4.c~completely-out-of-line-spinlocks--i386 arch/i386/oprofile/op_model_p4.c --- 25/arch/i386/oprofile/op_model_p4.c~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.591555952 -0700 +++ 25-akpm/arch/i386/oprofile/op_model_p4.c 2004-08-30 00:20:05.605553824 -0700 @@ -625,7 +625,7 @@ static int p4_check_ctrs(unsigned int co { unsigned long ctr, low, high, stag, real; int i; - unsigned long eip = instruction_pointer(regs); + unsigned long eip = profile_pc(regs); int is_kernel = !user_mode(regs); stag = get_stagger(); diff -puN arch/i386/oprofile/op_model_ppro.c~completely-out-of-line-spinlocks--i386 arch/i386/oprofile/op_model_ppro.c --- 25/arch/i386/oprofile/op_model_ppro.c~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.593555648 -0700 +++ 25-akpm/arch/i386/oprofile/op_model_ppro.c 2004-08-30 00:20:05.606553672 -0700 @@ -91,7 +91,7 @@ static int ppro_check_ctrs(unsigned int { unsigned int low, high; int i; - unsigned long eip = instruction_pointer(regs); + unsigned long eip = profile_pc(regs); int is_kernel = !user_mode(regs); for (i = 0 ; i < NUM_COUNTERS; ++i) { diff -puN include/asm-i386/ptrace.h~completely-out-of-line-spinlocks--i386 include/asm-i386/ptrace.h --- 25/include/asm-i386/ptrace.h~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.597555040 -0700 +++ 25-akpm/include/asm-i386/ptrace.h 2004-08-30 00:20:05.602554280 -0700 @@ -57,7 +57,11 @@ struct pt_regs { #ifdef __KERNEL__ #define user_mode(regs) ((VM_MASK & (regs)->eflags) || (3 & (regs)->xcs)) #define instruction_pointer(regs) ((regs)->eip) +#ifdef CONFIG_SMP +extern unsigned long profile_pc(struct pt_regs *regs); +#else #define profile_pc(regs) instruction_pointer(regs) #endif +#endif #endif diff -puN include/asm-i386/spinlock.h~completely-out-of-line-spinlocks--i386 include/asm-i386/spinlock.h --- 25/include/asm-i386/spinlock.h~completely-out-of-line-spinlocks--i386 2004-08-30 00:20:05.598554888 -0700 +++ 25-akpm/include/asm-i386/spinlock.h 2004-08-30 00:20:05.602554280 -0700 @@ -43,34 +43,10 @@ typedef struct { #define spin_is_locked(x) (*(volatile signed char *)(&(x)->lock) <= 0) #define spin_unlock_wait(x) do { barrier(); } while(spin_is_locked(x)) -#define spin_lock_string \ - "\n1:\t" \ - "lock ; decb %0\n\t" \ - "js 2f\n" \ - LOCK_SECTION_START("") \ - "2:\t" \ - "rep;nop\n\t" \ - "cmpb $0,%0\n\t" \ - "jle 2b\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END - -#define spin_lock_string_flags \ - "\n1:\t" \ - "lock ; decb %0\n\t" \ - "js 2f\n\t" \ - LOCK_SECTION_START("") \ - "2:\t" \ - "testl $0x200, %1\n\t" \ - "jz 3f\n\t" \ - "sti\n\t" \ - "3:\t" \ - "rep;nop\n\t" \ - "cmpb $0, %0\n\t" \ - "jle 3b\n\t" \ - "cli\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END +void fastcall __spin_lock_loop(spinlock_t *); +void fastcall __spin_lock_loop_flags(spinlock_t *, unsigned long); +extern unsigned long __spinlock_text_start; +extern unsigned long __spinlock_text_end; /* * This works. Despite all the confusion. @@ -137,12 +113,10 @@ here: BUG(); } #endif - __asm__ __volatile__( - spin_lock_string - :"=m" (lock->lock) : : "memory"); + __spin_lock_loop(lock); } -static inline void _raw_spin_lock_flags (spinlock_t *lock, unsigned long flags) +static inline void _raw_spin_lock_flags(spinlock_t *lock, unsigned long flags) { #ifdef CONFIG_DEBUG_SPINLOCK __label__ here; @@ -152,9 +126,7 @@ here: BUG(); } #endif - __asm__ __volatile__( - spin_lock_string_flags - :"=m" (lock->lock) : "r" (flags) : "memory"); + __spin_lock_loop_flags(lock, flags); } /* _