x86/asm/tsc: Add rdtsc_ordered() and use it in trivial call sites

rdtsc_barrier(); rdtsc() is an unnecessary mouthful and requires
more thought than should be necessary. Add an rdtsc_ordered()
helper and replace the trivial call sites with it.

This should not change generated code. The duplication of the
fence asm is temporary.

Signed-off-by: Andy Lutomirski <[email protected]>
Signed-off-by: Borislav Petkov <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Brian Gerst <[email protected]>
Cc: Denys Vlasenko <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Huang Rui <[email protected]>
Cc: John Stultz <[email protected]>
Cc: Len Brown <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Ralf Baechle <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: kvm ML <[email protected]>
Link: http://lkml.kernel.org/r/dddbf98a2af53312e9aa73a5a2b1622fe5d6f52b.1434501121.git.luto@kernel.org
Signed-off-by: Ingo Molnar <[email protected]>

diff --git a/arch/x86/entry/vdso/vclock_gettime.c b/arch/x86/entry/vdso/vclock_gettime.c
index 0340d93c18ca46e9bde33713e3e63a9281e18757..ca94fa6492516b0b1d669b6f52b09e968e3ae5cf 100644 (file)
--- a/arch/x86/entry/vdso/vclock_gettime.c
+++ b/arch/x86/entry/vdso/vclock_gettime.c
@@ -175,20 +175,8 @@ static notrace cycle_t vread_pvclock(int *mode)
 
 notrace static cycle_t vread_tsc(void)
 {
-       cycle_t ret;
-       u64 last;
-
-       /*
-        * Empirically, a fence (of type that depends on the CPU)
-        * before rdtsc is enough to ensure that rdtsc is ordered
-        * with respect to loads.  The various CPU manuals are unclear
-        * as to whether rdtsc can be reordered with later loads,
-        * but no one has ever seen it happen.
-        */
-       rdtsc_barrier();
-       ret = (cycle_t)rdtsc();
-
-       last = gtod->cycle_last;
+       cycle_t ret = (cycle_t)rdtsc_ordered();
+       u64 last = gtod->cycle_last;
 
        if (likely(ret >= last))
                return ret;

diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index ff0c120dafe5416cfe3dab2917402f2669fe77ac..02bdd6c65017006e4dbdedd0ac57859f70651263 100644 (file)
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -127,6 +127,32 @@ static __always_inline unsigned long long rdtsc(void)
        return EAX_EDX_VAL(val, low, high);
 }
 
+/**
+ * rdtsc_ordered() - read the current TSC in program order
+ *
+ * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
+ * It is ordered like a load to a global in-memory counter.  It should
+ * be impossible to observe non-monotonic rdtsc_unordered() behavior
+ * across multiple CPUs as long as the TSC is synced.
+ */
+static __always_inline unsigned long long rdtsc_ordered(void)
+{
+       /*
+        * The RDTSC instruction is not ordered relative to memory
+        * access.  The Intel SDM and the AMD APM are both vague on this
+        * point, but empirically an RDTSC instruction can be
+        * speculatively executed before prior loads.  An RDTSC
+        * immediately after an appropriate barrier appears to be
+        * ordered as a normal load, that is, it provides the same
+        * ordering guarantees as reading from a global memory location
+        * that some other imaginary CPU is updating continuously with a
+        * time stamp.
+        */
+       alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC,
+                         "lfence", X86_FEATURE_LFENCE_RDTSC);
+       return rdtsc();
+}
+
 static inline unsigned long long native_read_pmc(int counter)
 {
        DECLARE_ARGS(val, low, high);

diff --git a/arch/x86/kernel/trace_clock.c b/arch/x86/kernel/trace_clock.c
index 67efb8c96fc4c6a73ad60380a816a56efffa3aba..80bb24d9b880cd257b2daa2e8ea5aea3aaf9ea12 100644 (file)
--- a/arch/x86/kernel/trace_clock.c
+++ b/arch/x86/kernel/trace_clock.c
@@ -12,10 +12,5 @@
  */
 u64 notrace trace_clock_x86_tsc(void)
 {
-       u64 ret;
-
-       rdtsc_barrier();
-       ret = rdtsc();
-
-       return ret;
+       return rdtsc_ordered();
 }

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index dfa97139282db4eb02735e9d8f6a23c13135f5a0..8d73ec8a236437af4690e67ad56493cd4b06f881 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1444,20 +1444,8 @@ EXPORT_SYMBOL_GPL(kvm_write_tsc);
 
 static cycle_t read_tsc(void)
 {
-       cycle_t ret;
-       u64 last;
-
-       /*
-        * Empirically, a fence (of type that depends on the CPU)
-        * before rdtsc is enough to ensure that rdtsc is ordered
-        * with respect to loads.  The various CPU manuals are unclear
-        * as to whether rdtsc can be reordered with later loads,
-        * but no one has ever seen it happen.
-        */
-       rdtsc_barrier();
-       ret = (cycle_t)rdtsc();
-
-       last = pvclock_gtod_data.clock.cycle_last;
+       cycle_t ret = (cycle_t)rdtsc_ordered();
+       u64 last = pvclock_gtod_data.clock.cycle_last;
 
        if (likely(ret >= last))
                return ret;

diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
index f24bc59ab0a0ac8261769ac49d6875025d1db480..4453d52a143d6645c1c19bb99149df8fb667b65d 100644 (file)
--- a/arch/x86/lib/delay.c
+++ b/arch/x86/lib/delay.c
@@ -54,11 +54,9 @@ static void delay_tsc(unsigned long __loops)
 
        preempt_disable();
        cpu = smp_processor_id();
-       rdtsc_barrier();
-       bclock = rdtsc();
+       bclock = rdtsc_ordered();
        for (;;) {
-               rdtsc_barrier();
-               now = rdtsc();
+               now = rdtsc_ordered();
                if ((now - bclock) >= loops)
                        break;
 
@@ -79,8 +77,7 @@ static void delay_tsc(unsigned long __loops)
                if (unlikely(cpu != smp_processor_id())) {
                        loops -= (now - bclock);
                        cpu = smp_processor_id();
-                       rdtsc_barrier();
-                       bclock = rdtsc();
+                       bclock = rdtsc_ordered();
                }
        }
        preempt_enable();