From 751de83c0c94a5235f14cff8549d3b39e745eb2b Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:03 +0100 Subject: x86: unify msr smp funcs The functions under #ifdef CONFIG_SMP in msr.h are the same for both x86_64 and i386, and this patches removes one of them, putting them in a single location Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 80b027081b3c..5c95d0be1132 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -139,29 +139,6 @@ static inline int wrmsr_safe(u32 __msr, u32 __low, u32 __high) } while(0) #endif /* !CONFIG_PARAVIRT */ -#ifdef CONFIG_SMP -void rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); -void wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); -int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); -int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); -#else /* CONFIG_SMP */ -static inline void rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) -{ - rdmsr(msr_no, *l, *h); -} -static inline void wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) -{ - wrmsr(msr_no, l, h); -} -static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) -{ - return rdmsr_safe(msr_no, l, h); -} -static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) -{ - return wrmsr_safe(msr_no, l, h); -} -#endif /* CONFIG_SMP */ #endif /* ! __ASSEMBLY__ */ #endif /* __KERNEL__ */ @@ -327,6 +304,12 @@ static inline unsigned int cpuid_edx(unsigned int op) :"c"(msr), "i"(-EIO), "0"(0)); \ ret__; }) +#endif /* __ASSEMBLY__ */ + +#endif /* !__i386__ */ + +#ifndef __ASSEMBLY__ + #ifdef CONFIG_SMP void rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); void wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); @@ -351,8 +334,6 @@ static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) } #endif /* CONFIG_SMP */ #endif /* __KERNEL__ */ -#endif /* __ASSEMBLY__ */ - -#endif /* !__i386__ */ +#endif /* __ASSEMBLY__ */ #endif -- cgit v1.2.3 From c758ecf62ad94ddfeb4e7d8a5498bdcb2e3c85db Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:03 +0100 Subject: x86: unify cpuid functions cpuid is not very different between i386 and x86_64. We move away the x86_64 version from msr.h, and unify them at processor.h, where they belong. cpuid() paravirt then comes for free. Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 67 ------------------------------------ include/asm-x86/processor.h | 78 ++++++++++++++++++++++++++++++++++++++++++ include/asm-x86/processor_32.h | 69 ------------------------------------- 3 files changed, 78 insertions(+), 136 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 5c95d0be1132..0648b2d57a38 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -203,73 +203,6 @@ static inline int wrmsr_safe(u32 __msr, u32 __low, u32 __high) : "c" (counter)) -static inline void cpuid(int op, unsigned int *eax, unsigned int *ebx, - unsigned int *ecx, unsigned int *edx) -{ - __asm__("cpuid" - : "=a" (*eax), - "=b" (*ebx), - "=c" (*ecx), - "=d" (*edx) - : "0" (op)); -} - -/* Some CPUID calls want 'count' to be placed in ecx */ -static inline void cpuid_count(int op, int count, int *eax, int *ebx, int *ecx, - int *edx) -{ - __asm__("cpuid" - : "=a" (*eax), - "=b" (*ebx), - "=c" (*ecx), - "=d" (*edx) - : "0" (op), "c" (count)); -} - -/* - * CPUID functions returning a single datum - */ -static inline unsigned int cpuid_eax(unsigned int op) -{ - unsigned int eax; - - __asm__("cpuid" - : "=a" (eax) - : "0" (op) - : "bx", "cx", "dx"); - return eax; -} -static inline unsigned int cpuid_ebx(unsigned int op) -{ - unsigned int eax, ebx; - - __asm__("cpuid" - : "=a" (eax), "=b" (ebx) - : "0" (op) - : "cx", "dx" ); - return ebx; -} -static inline unsigned int cpuid_ecx(unsigned int op) -{ - unsigned int eax, ecx; - - __asm__("cpuid" - : "=a" (eax), "=c" (ecx) - : "0" (op) - : "bx", "dx" ); - return ecx; -} -static inline unsigned int cpuid_edx(unsigned int op) -{ - unsigned int eax, edx; - - __asm__("cpuid" - : "=a" (eax), "=d" (edx) - : "0" (op) - : "bx", "cx"); - return edx; -} - #ifdef __KERNEL__ /* wrmsr with exception handling */ diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h index 46e1c04e309c..dea81b70895d 100644 --- a/include/asm-x86/processor.h +++ b/include/asm-x86/processor.h @@ -1,5 +1,83 @@ +#ifndef __ASM_X86_PROCESSOR_H +#define __ASM_X86_PROCESSOR_H + +static inline void native_cpuid(unsigned int *eax, unsigned int *ebx, + unsigned int *ecx, unsigned int *edx) +{ + /* ecx is often an input as well as an output. */ + __asm__("cpuid" + : "=a" (*eax), + "=b" (*ebx), + "=c" (*ecx), + "=d" (*edx) + : "0" (*eax), "2" (*ecx)); +} + + #ifdef CONFIG_X86_32 # include "processor_32.h" #else # include "processor_64.h" #endif + +#ifndef CONFIG_PARAVIRT +#define __cpuid native_cpuid +#endif + +/* + * Generic CPUID function + * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx + * resulting in stale register contents being returned. + */ +static inline void cpuid(unsigned int op, + unsigned int *eax, unsigned int *ebx, + unsigned int *ecx, unsigned int *edx) +{ + *eax = op; + *ecx = 0; + __cpuid(eax, ebx, ecx, edx); +} + +/* Some CPUID calls want 'count' to be placed in ecx */ +static inline void cpuid_count(unsigned int op, int count, + unsigned int *eax, unsigned int *ebx, + unsigned int *ecx, unsigned int *edx) +{ + *eax = op; + *ecx = count; + __cpuid(eax, ebx, ecx, edx); +} + +/* + * CPUID functions returning a single datum + */ +static inline unsigned int cpuid_eax(unsigned int op) +{ + unsigned int eax, ebx, ecx, edx; + + cpuid(op, &eax, &ebx, &ecx, &edx); + return eax; +} +static inline unsigned int cpuid_ebx(unsigned int op) +{ + unsigned int eax, ebx, ecx, edx; + + cpuid(op, &eax, &ebx, &ecx, &edx); + return ebx; +} +static inline unsigned int cpuid_ecx(unsigned int op) +{ + unsigned int eax, ebx, ecx, edx; + + cpuid(op, &eax, &ebx, &ecx, &edx); + return ecx; +} +static inline unsigned int cpuid_edx(unsigned int op) +{ + unsigned int eax, ebx, ecx, edx; + + cpuid(op, &eax, &ebx, &ecx, &edx); + return edx; +} + +#endif diff --git a/include/asm-x86/processor_32.h b/include/asm-x86/processor_32.h index 6846cc346400..0d83da198127 100644 --- a/include/asm-x86/processor_32.h +++ b/include/asm-x86/processor_32.h @@ -133,18 +133,6 @@ extern void detect_ht(struct cpuinfo_x86 *c); static inline void detect_ht(struct cpuinfo_x86 *c) {} #endif -static inline void native_cpuid(unsigned int *eax, unsigned int *ebx, - unsigned int *ecx, unsigned int *edx) -{ - /* ecx is often an input as well as an output. */ - __asm__("cpuid" - : "=a" (*eax), - "=b" (*ebx), - "=c" (*ecx), - "=d" (*edx) - : "0" (*eax), "2" (*ecx)); -} - #define load_cr3(pgdir) write_cr3(__pa(pgdir)) /* @@ -580,7 +568,6 @@ static inline void native_set_iopl_mask(unsigned mask) #include #else #define paravirt_enabled() 0 -#define __cpuid native_cpuid static inline void load_sp0(struct tss_struct *tss, struct thread_struct *thread) { @@ -598,62 +585,6 @@ static inline void load_sp0(struct tss_struct *tss, struct thread_struct *thread #define set_iopl_mask native_set_iopl_mask #endif /* CONFIG_PARAVIRT */ -/* - * Generic CPUID function - * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx - * resulting in stale register contents being returned. - */ -static inline void cpuid(unsigned int op, - unsigned int *eax, unsigned int *ebx, - unsigned int *ecx, unsigned int *edx) -{ - *eax = op; - *ecx = 0; - __cpuid(eax, ebx, ecx, edx); -} - -/* Some CPUID calls want 'count' to be placed in ecx */ -static inline void cpuid_count(unsigned int op, int count, - unsigned int *eax, unsigned int *ebx, - unsigned int *ecx, unsigned int *edx) -{ - *eax = op; - *ecx = count; - __cpuid(eax, ebx, ecx, edx); -} - -/* - * CPUID functions returning a single datum - */ -static inline unsigned int cpuid_eax(unsigned int op) -{ - unsigned int eax, ebx, ecx, edx; - - cpuid(op, &eax, &ebx, &ecx, &edx); - return eax; -} -static inline unsigned int cpuid_ebx(unsigned int op) -{ - unsigned int eax, ebx, ecx, edx; - - cpuid(op, &eax, &ebx, &ecx, &edx); - return ebx; -} -static inline unsigned int cpuid_ecx(unsigned int op) -{ - unsigned int eax, ebx, ecx, edx; - - cpuid(op, &eax, &ebx, &ecx, &edx); - return ecx; -} -static inline unsigned int cpuid_edx(unsigned int op) -{ - unsigned int eax, ebx, ecx, edx; - - cpuid(op, &eax, &ebx, &ecx, &edx); - return edx; -} - /* generic versions from gas */ #define GENERIC_NOP1 ".byte 0x90\n" #define GENERIC_NOP2 ".byte 0x89,0xf6\n" -- cgit v1.2.3 From 8f12dea6135d0a55b151dcb4c6bbe211f5f8d35d Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:06 +0100 Subject: x86: introduce native_read_tscp Targetting paravirt, this patch introduces native_read_tscp, in place of rdtscp() macro. When in a paravirt guest, this will involve a function call, and thus, cannot be done in the vdso area. These users then have to call the native version directly Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/vsyscall_64.c | 4 ++-- arch/x86/vdso/vgetcpu.c | 4 ++-- include/asm-x86/msr.h | 29 +++++++++++++++++++++-------- 3 files changed, 25 insertions(+), 12 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c index 018f7cf33790..c4c5e765cd2c 100644 --- a/arch/x86/kernel/vsyscall_64.c +++ b/arch/x86/kernel/vsyscall_64.c @@ -190,7 +190,7 @@ time_t __vsyscall(1) vtime(time_t *t) long __vsyscall(2) vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache) { - unsigned int dummy, p; + unsigned int p; unsigned long j = 0; /* Fast cache - only recompute value once per jiffies and avoid @@ -205,7 +205,7 @@ vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache) p = tcache->blob[1]; } else if (__vgetcpu_mode == VGETCPU_RDTSCP) { /* Load per CPU data from RDTSCP */ - rdtscp(dummy, dummy, p); + native_read_tscp(&p); } else { /* Load per CPU data from GDT */ asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); diff --git a/arch/x86/vdso/vgetcpu.c b/arch/x86/vdso/vgetcpu.c index 3b1ae1abfba9..c8097f17f8a9 100644 --- a/arch/x86/vdso/vgetcpu.c +++ b/arch/x86/vdso/vgetcpu.c @@ -15,11 +15,11 @@ long __vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) { - unsigned int dummy, p; + unsigned int p; if (*vdso_vgetcpu_mode == VGETCPU_RDTSCP) { /* Load per CPU data from RDTSCP */ - rdtscp(dummy, dummy, p); + native_read_tscp(&p); } else { /* Load per CPU data from GDT */ asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 0648b2d57a38..b6262e99fc8e 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -7,6 +7,27 @@ # include #endif +#ifdef __KERNEL__ +#ifndef __ASSEMBLY__ +static inline unsigned long long native_read_tscp(int *aux) +{ + unsigned long low, high; + asm volatile (".byte 0x0f,0x01,0xf9" + : "=a" (low), "=d" (high), "=c" (*aux)); + return low | ((u64)high >> 32); +} + +#define rdtscp(low, high, aux) \ + do { \ + unsigned long long _val = native_read_tscp(&(aux)); \ + (low) = (u32)_val; \ + (high) = (u32)(_val >> 32); \ + } while (0) + +#define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) +#endif +#endif + #ifdef __i386__ #ifdef __KERNEL__ @@ -178,8 +199,6 @@ static inline int wrmsr_safe(u32 __msr, u32 __low, u32 __high) #define rdtscl(low) \ __asm__ __volatile__ ("rdtsc" : "=a" (low) : : "edx") -#define rdtscp(low,high,aux) \ - __asm__ __volatile__ (".byte 0x0f,0x01,0xf9" : "=a" (low), "=d" (high), "=c" (aux)) #define rdtscll(val) do { \ unsigned int __a,__d; \ @@ -187,12 +206,6 @@ static inline int wrmsr_safe(u32 __msr, u32 __low, u32 __high) (val) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \ } while(0) -#define rdtscpll(val, aux) do { \ - unsigned long __a, __d; \ - __asm__ __volatile__ (".byte 0x0f,0x01,0xf9" : "=a" (__a), "=d" (__d), "=c" (aux)); \ - (val) = (__d << 32) | __a; \ -} while (0) - #define write_tsc(val1,val2) wrmsr(0x10, val1, val2) #define write_rdtscp_aux(val) wrmsr(0xc0000103, val, 0) -- cgit v1.2.3 From b8d1fae7dbde6a1227fa142acecb48dc3dd63817 Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:07 +0100 Subject: x86: change rdpmc interface the rdpmc instruction gets a counter argument in rcx. However, the i386 version was ignoring it. To make both x86_64 and i386 versions the same, as well as to comply with the instruction semantics, this parameter is added in the i386 version Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 6 +++--- include/asm-x86/paravirt.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index b6262e99fc8e..effb7319c0a5 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -94,10 +94,10 @@ static inline unsigned long long native_read_tsc(void) return val; } -static inline unsigned long long native_read_pmc(void) +static inline unsigned long long native_read_pmc(int counter) { unsigned long long val; - asm volatile("rdpmc" : "=A" (val)); + asm volatile("rdpmc" : "=A" (val) : "c" (counter)); return val; } @@ -154,7 +154,7 @@ static inline int wrmsr_safe(u32 __msr, u32 __low, u32 __high) #define rdpmc(counter,low,high) \ do { \ - u64 _l = native_read_pmc(); \ + u64 _l = native_read_pmc(counter); \ (low) = (u32)_l; \ (high) = (u32)(_l >> 32); \ } while(0) diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index d1780e32722e..4782be68d0fa 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -118,7 +118,7 @@ struct pv_cpu_ops { int (*write_msr)(unsigned int msr, u64 val); u64 (*read_tsc)(void); - u64 (*read_pmc)(void); + u64 (*read_pmc)(int counter); /* These two are jmp to, not actually called. */ void (*irq_enable_syscall_ret)(void); -- cgit v1.2.3 From c9dcda5ce46c395c5c99003e259e1973dface640 Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:07 +0100 Subject: x86: change write msr functions interface This patche changes the native_write_msr() and friends interface to explicitly take 2 32-bit registers instead of a 64-bit value. The change will ease the merge with 64-bit code. As the 64-bit value will be passed as two registers anyway in i386, the PVOP_CALL interface has to account for that and use low/high parameters It would force the x86_64 version to be different. The change does not make i386 generated code less efficient. As said above, it would get the values from two registers anyway. Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 19 ++++++++++--------- include/asm-x86/paravirt.h | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index effb7319c0a5..cb7222358897 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -63,13 +63,14 @@ static inline unsigned long long native_read_msr_safe(unsigned int msr, return val; } -static inline void native_write_msr(unsigned int msr, unsigned long long val) +static inline void native_write_msr(unsigned int msr, + unsigned low, unsigned high) { - asm volatile("wrmsr" : : "c" (msr), "A"(val)); + asm volatile("wrmsr" : : "c" (msr), "a"(low), "d" (high)); } static inline int native_write_msr_safe(unsigned int msr, - unsigned long long val) + unsigned low, unsigned high) { int err; asm volatile("2: wrmsr ; xorl %0,%0\n" @@ -82,7 +83,7 @@ static inline int native_write_msr_safe(unsigned int msr, " .long 2b,3b\n\t" ".previous" : "=a" (err) - : "c" (msr), "0" ((u32)val), "d" ((u32)(val>>32)), + : "c" (msr), "0" (low), "d" (high), "i" (-EFAULT)); return err; } @@ -118,20 +119,20 @@ static inline unsigned long long native_read_pmc(int counter) (val2) = (u32)(__val >> 32); \ } while(0) -static inline void wrmsr(u32 __msr, u32 __low, u32 __high) +static inline void wrmsr(unsigned msr, unsigned low, unsigned high) { - native_write_msr(__msr, ((u64)__high << 32) | __low); + native_write_msr(msr, low, high); } #define rdmsrl(msr,val) \ ((val) = native_read_msr(msr)) -#define wrmsrl(msr,val) native_write_msr(msr, val) +#define wrmsrl(msr, val) native_write_msr(msr, (u32)val, (u32)(val >> 32)) /* wrmsr with exception handling */ -static inline int wrmsr_safe(u32 __msr, u32 __low, u32 __high) +static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) { - return native_write_msr_safe(__msr, ((u64)__high << 32) | __low); + return native_write_msr_safe(msr, low, high); } /* rdmsr with exception handling */ diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index 4782be68d0fa..e95c2a655165 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -115,7 +115,7 @@ struct pv_cpu_ops { /* MSR, PMC and TSR operations. err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */ u64 (*read_msr)(unsigned int msr, int *err); - int (*write_msr)(unsigned int msr, u64 val); + int (*write_msr)(unsigned int msr, unsigned low, unsigned high); u64 (*read_tsc)(void); u64 (*read_pmc)(int counter); -- cgit v1.2.3 From 56ec1ddcff967e51d98427e4efcbfc90de67efe3 Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:07 +0100 Subject: x86: make fixups wordsize agnostic This patch uses the _ASM_ALIGN and _ASM_PTR macros to make the fixups in native_read/write_msr_safe look the same for x86_64 and i386. Besides using this macros, we also have to take the explicit instruction suffixes out. It's okay because all this instructions uses registers, and can be sized by them. Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index cb7222358897..792fde2e8908 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -33,6 +33,7 @@ static inline unsigned long long native_read_tscp(int *aux) #ifdef __KERNEL__ #ifndef __ASSEMBLY__ +#include #include static inline unsigned long long native_read_msr(unsigned int msr) @@ -48,14 +49,14 @@ static inline unsigned long long native_read_msr_safe(unsigned int msr, { unsigned long long val; - asm volatile("2: rdmsr ; xorl %0,%0\n" + asm volatile("2: rdmsr ; xor %0,%0\n" "1:\n\t" ".section .fixup,\"ax\"\n\t" - "3: movl %3,%0 ; jmp 1b\n\t" + "3: mov %3,%0 ; jmp 1b\n\t" ".previous\n\t" ".section __ex_table,\"a\"\n" - " .align 4\n\t" - " .long 2b,3b\n\t" + _ASM_ALIGN "\n\t" + _ASM_PTR " 2b,3b\n\t" ".previous" : "=r" (*err), "=A" (val) : "c" (msr), "i" (-EFAULT)); @@ -73,14 +74,14 @@ static inline int native_write_msr_safe(unsigned int msr, unsigned low, unsigned high) { int err; - asm volatile("2: wrmsr ; xorl %0,%0\n" + asm volatile("2: wrmsr ; xor %0,%0\n" "1:\n\t" ".section .fixup,\"ax\"\n\t" - "3: movl %4,%0 ; jmp 1b\n\t" + "3: mov %4,%0 ; jmp 1b\n\t" ".previous\n\t" ".section __ex_table,\"a\"\n" - " .align 4\n\t" - " .long 2b,3b\n\t" + _ASM_ALIGN "\n\t" + _ASM_PTR " 2b,3b\n\t" ".previous" : "=a" (err) : "c" (msr), "0" (low), "d" (high), -- cgit v1.2.3 From c210d24986dc19e387c10508c0bc2faadadc1a2e Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:31:07 +0100 Subject: x86: integrate 32-bit and 64-bit code in msr.h This patches proceeds with the integration of msr.h, making the code unified, instead of having a version for each architecture. We stick with the native_* functions, and then paravirt comes for free. Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 167 ++++++++++++++------------------------------------ 1 file changed, 47 insertions(+), 120 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 792fde2e8908..040d3910d891 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -9,6 +9,10 @@ #ifdef __KERNEL__ #ifndef __ASSEMBLY__ + +#include +#include + static inline unsigned long long native_read_tscp(int *aux) { unsigned long low, high; @@ -17,37 +21,36 @@ static inline unsigned long long native_read_tscp(int *aux) return low | ((u64)high >> 32); } -#define rdtscp(low, high, aux) \ - do { \ - unsigned long long _val = native_read_tscp(&(aux)); \ - (low) = (u32)_val; \ - (high) = (u32)(_val >> 32); \ - } while (0) - -#define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) -#endif +/* + * i386 calling convention returns 64-bit value in edx:eax, while + * x86_64 returns at rax. Also, the "A" constraint does not really + * mean rdx:rax in x86_64, so we need specialized behaviour for each + * architecture + */ +#ifdef CONFIG_X86_64 +#define DECLARE_ARGS(val, low, high) unsigned low, high +#define EAX_EDX_VAL(val, low, high) (low | ((u64)(high) << 32)) +#define EAX_EDX_ARGS(val, low, high) "a" (low), "d" (high) +#define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) +#else +#define DECLARE_ARGS(val, low, high) unsigned long long val +#define EAX_EDX_VAL(val, low, high) (val) +#define EAX_EDX_ARGS(val, low, high) "A" (val) +#define EAX_EDX_RET(val, low, high) "=A" (val) #endif -#ifdef __i386__ - -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ - -#include -#include - static inline unsigned long long native_read_msr(unsigned int msr) { - unsigned long long val; + DECLARE_ARGS(val, low, high); - asm volatile("rdmsr" : "=A" (val) : "c" (msr)); - return val; + asm volatile("rdmsr" : EAX_EDX_RET(val, low, high) : "c" (msr)); + return EAX_EDX_VAL(val, low, high); } static inline unsigned long long native_read_msr_safe(unsigned int msr, int *err) { - unsigned long long val; + DECLARE_ARGS(val, low, high); asm volatile("2: rdmsr ; xor %0,%0\n" "1:\n\t" @@ -58,10 +61,9 @@ static inline unsigned long long native_read_msr_safe(unsigned int msr, _ASM_ALIGN "\n\t" _ASM_PTR " 2b,3b\n\t" ".previous" - : "=r" (*err), "=A" (val) + : "=r" (*err), EAX_EDX_RET(val, low, high) : "c" (msr), "i" (-EFAULT)); - - return val; + return EAX_EDX_VAL(val, low, high); } static inline void native_write_msr(unsigned int msr, @@ -91,16 +93,18 @@ static inline int native_write_msr_safe(unsigned int msr, static inline unsigned long long native_read_tsc(void) { - unsigned long long val; - asm volatile("rdtsc" : "=A" (val)); - return val; + DECLARE_ARGS(val, low, high); + + asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); + return EAX_EDX_VAL(val, low, high); } static inline unsigned long long native_read_pmc(int counter) { - unsigned long long val; - asm volatile("rdpmc" : "=A" (val) : "c" (counter)); - return val; + DECLARE_ARGS(val, low, high); + + asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); + return EAX_EDX_VAL(val, low, high); } #ifdef CONFIG_PARAVIRT @@ -128,7 +132,8 @@ static inline void wrmsr(unsigned msr, unsigned low, unsigned high) #define rdmsrl(msr,val) \ ((val) = native_read_msr(msr)) -#define wrmsrl(msr, val) native_write_msr(msr, (u32)val, (u32)(val >> 32)) +#define wrmsrl(msr, val) \ + native_write_msr(msr, (u32)((u64)(val)), (u32)((u64)(val) >> 32)) /* wrmsr with exception handling */ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) @@ -160,104 +165,25 @@ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) (low) = (u32)_l; \ (high) = (u32)(_l >> 32); \ } while(0) -#endif /* !CONFIG_PARAVIRT */ - -#endif /* ! __ASSEMBLY__ */ -#endif /* __KERNEL__ */ - -#else /* __i386__ */ - -#ifndef __ASSEMBLY__ -#include -/* - * Access to machine-specific registers (available on 586 and better only) - * Note: the rd* operations modify the parameters directly (without using - * pointer indirection), this allows gcc to optimize better - */ - -#define rdmsr(msr,val1,val2) \ - __asm__ __volatile__("rdmsr" \ - : "=a" (val1), "=d" (val2) \ - : "c" (msr)) - -#define rdmsrl(msr,val) do { unsigned long a__,b__; \ - __asm__ __volatile__("rdmsr" \ - : "=a" (a__), "=d" (b__) \ - : "c" (msr)); \ - val = a__ | (b__<<32); \ -} while(0) - -#define wrmsr(msr,val1,val2) \ - __asm__ __volatile__("wrmsr" \ - : /* no outputs */ \ - : "c" (msr), "a" (val1), "d" (val2)) - -#define wrmsrl(msr,val) wrmsr(msr,(__u32)((__u64)(val)),((__u64)(val))>>32) +#define rdtscp(low, high, aux) \ + do { \ + unsigned long long _val = native_read_tscp(&(aux)); \ + (low) = (u32)_val; \ + (high) = (u32)(_val >> 32); \ + } while (0) -#define rdtsc(low,high) \ - __asm__ __volatile__("rdtsc" : "=a" (low), "=d" (high)) +#define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) -#define rdtscl(low) \ - __asm__ __volatile__ ("rdtsc" : "=a" (low) : : "edx") +#endif /* !CONFIG_PARAVIRT */ -#define rdtscll(val) do { \ - unsigned int __a,__d; \ - __asm__ __volatile__("rdtsc" : "=a" (__a), "=d" (__d)); \ - (val) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \ -} while(0) +#define checking_wrmsrl(msr,val) wrmsr_safe(msr,(u32)(val),(u32)((val)>>32)) #define write_tsc(val1,val2) wrmsr(0x10, val1, val2) #define write_rdtscp_aux(val) wrmsr(0xc0000103, val, 0) -#define rdpmc(counter,low,high) \ - __asm__ __volatile__("rdpmc" \ - : "=a" (low), "=d" (high) \ - : "c" (counter)) - - -#ifdef __KERNEL__ - -/* wrmsr with exception handling */ -#define wrmsr_safe(msr,a,b) ({ int ret__; \ - asm volatile("2: wrmsr ; xorl %0,%0\n" \ - "1:\n\t" \ - ".section .fixup,\"ax\"\n\t" \ - "3: movl %4,%0 ; jmp 1b\n\t" \ - ".previous\n\t" \ - ".section __ex_table,\"a\"\n" \ - " .align 8\n\t" \ - " .quad 2b,3b\n\t" \ - ".previous" \ - : "=a" (ret__) \ - : "c" (msr), "0" (a), "d" (b), "i" (-EFAULT)); \ - ret__; }) - -#define checking_wrmsrl(msr,val) wrmsr_safe(msr,(u32)(val),(u32)((val)>>32)) - -#define rdmsr_safe(msr,a,b) \ - ({ int ret__; \ - asm volatile ("1: rdmsr\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl %4,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 8\n" \ - " .quad 1b,3b\n" \ - ".previous":"=&bDS" (ret__), "=a"(*(a)), "=d"(*(b)) \ - :"c"(msr), "i"(-EIO), "0"(0)); \ - ret__; }) - -#endif /* __ASSEMBLY__ */ - -#endif /* !__i386__ */ - -#ifndef __ASSEMBLY__ - #ifdef CONFIG_SMP void rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); void wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); @@ -281,7 +207,8 @@ static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) return wrmsr_safe(msr_no, l, h); } #endif /* CONFIG_SMP */ -#endif /* __KERNEL__ */ #endif /* __ASSEMBLY__ */ +#endif /* __KERNEL__ */ + #endif -- cgit v1.2.3 From 1e160cc3f3b5c38bba79c58172e82c1e24934546 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Wed, 30 Jan 2008 13:31:17 +0100 Subject: x86: __vdso_getcpu() warning fix arch/x86/vdso/vgetcpu.c: In function '__vdso_getcpu': arch/x86/vdso/vgetcpu.c:22: warning: pointer targets in passing argument 1 of 'native_read_tscp' differ in signedness Signed-off-by: Andrew Morton Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 040d3910d891..088652a62d58 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -13,7 +13,7 @@ #include #include -static inline unsigned long long native_read_tscp(int *aux) +static inline unsigned long long native_read_tscp(unsigned int *aux) { unsigned long low, high; asm volatile (".byte 0x0f,0x01,0xf9" -- cgit v1.2.3 From f72a9ef979c5a828c64deb88ebba743f7d899907 Mon Sep 17 00:00:00 2001 From: Glauber de Oliveira Costa Date: Wed, 30 Jan 2008 13:32:05 +0100 Subject: x86: cleanup write_tsc write_tsc() does not need to be enclosed in any paravirt closure, as it uses wrmsr(). So we rip off the duplicate in msr.h and the definition from paravirt.h Signed-off-by: Glauber de Oliveira Costa Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- include/asm-x86/msr.h | 2 -- include/asm-x86/paravirt.h | 2 -- 2 files changed, 4 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index 088652a62d58..bca8c3950132 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -157,8 +157,6 @@ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) #define rdtscll(val) \ ((val) = native_read_tsc()) -#define write_tsc(val1,val2) wrmsr(0x10, val1, val2) - #define rdpmc(counter,low,high) \ do { \ u64 _l = native_read_pmc(counter); \ diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index bef932772661..66290f58e939 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -657,8 +657,6 @@ static inline unsigned long long paravirt_sched_clock(void) } #define calculate_cpu_khz() (pv_time_ops.get_cpu_khz()) -#define write_tsc(val1,val2) wrmsr(0x10, val1, val2) - static inline unsigned long long paravirt_read_pmc(int counter) { return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter); -- cgit v1.2.3 From cdc7957d1954908a39a6964e9c6f643916e76c4b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 30 Jan 2008 13:32:39 +0100 Subject: x86: move native_read_tsc() offline move native_read_tsc() offline. Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/rtc.c | 10 ++++++++++ include/asm-x86/msr.h | 8 +------- 2 files changed, 11 insertions(+), 7 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index d040840ff1b6..c42cf1263882 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -6,6 +6,7 @@ #include #include +#include #ifdef CONFIG_X86_32 # define CMOS_YEARS_OFFS 1900 @@ -194,3 +195,12 @@ int update_persistent_clock(struct timespec now) { return set_rtc_mmss(now.tv_sec); } + +unsigned long long __vsyscall_fn native_read_tsc(void) +{ + DECLARE_ARGS(val, low, high); + + asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); + return EAX_EDX_VAL(val, low, high); +} +EXPORT_SYMBOL_GPL(native_read_tsc); diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index bca8c3950132..decfec4ab17c 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -91,13 +91,7 @@ static inline int native_write_msr_safe(unsigned int msr, return err; } -static inline unsigned long long native_read_tsc(void) -{ - DECLARE_ARGS(val, low, high); - - asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); - return EAX_EDX_VAL(val, low, high); -} +extern unsigned long long native_read_tsc(void); static inline unsigned long long native_read_pmc(int counter) { -- cgit v1.2.3 From 92767af0e3904b4d35ed547fb514ff6cb227e678 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 30 Jan 2008 13:32:40 +0100 Subject: x86: fix sched_clock() [ andi@firstfloor.org: build fix ] Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- arch/x86/kernel/rtc.c | 13 ++++--------- include/asm-x86/msr.h | 11 +++++++++++ include/asm-x86/tsc.h | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) (limited to 'include/asm-x86/msr.h') diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index 276cb7073ab1..eb9b1a198f5e 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -196,14 +196,9 @@ int update_persistent_clock(struct timespec now) return set_rtc_mmss(now.tv_sec); } -unsigned long long __vsyscall_fn native_read_tsc(void) +unsigned long long native_read_tsc(void) { - DECLARE_ARGS(val, low, high); - - rdtsc_barrier(); - asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); - rdtsc_barrier(); - - return EAX_EDX_VAL(val, low, high); + return __native_read_tsc(); } -EXPORT_SYMBOL_GPL(native_read_tsc); +EXPORT_SYMBOL(native_read_tsc); + diff --git a/include/asm-x86/msr.h b/include/asm-x86/msr.h index decfec4ab17c..204a8a30fecf 100644 --- a/include/asm-x86/msr.h +++ b/include/asm-x86/msr.h @@ -93,6 +93,17 @@ static inline int native_write_msr_safe(unsigned int msr, extern unsigned long long native_read_tsc(void); +static __always_inline unsigned long long __native_read_tsc(void) +{ + DECLARE_ARGS(val, low, high); + + rdtsc_barrier(); + asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); + rdtsc_barrier(); + + return EAX_EDX_VAL(val, low, high); +} + static inline unsigned long long native_read_pmc(int counter) { DECLARE_ARGS(val, low, high); diff --git a/include/asm-x86/tsc.h b/include/asm-x86/tsc.h index f51a50da35aa..071e0ce5b664 100644 --- a/include/asm-x86/tsc.h +++ b/include/asm-x86/tsc.h @@ -46,7 +46,7 @@ static inline cycles_t vget_cycles(void) if (!cpu_has_tsc) return 0; #endif - return (cycles_t) native_read_tsc(); + return (cycles_t) __native_read_tsc(); } extern void tsc_init(void); -- cgit v1.2.3