File 57a1e64c-x86-time-introduce-and-use-rdtsc_ordered.patch of Package xen.6117
References: bsc#970135
# Commit fa74e70500fd73dd2fc441c7dc00b190fb37cee5
# Date 2016-08-03 14:40:44 +0200
# Author Jan Beulich <jbeulich@suse.com>
# Committer Jan Beulich <jbeulich@suse.com>
x86/time: introduce and use rdtsc_ordered()
Matching Linux commit 03b9730b76 ("x86/asm/tsc: Add rdtsc_ordered() and
use it in trivial call sites") and earlier ones it builds upon, let's
make sure timing loops don't have their rdtsc()-s re-ordered, as that
would harm precision of the result (values were observed to be several
hundred clocks off without this adjustment).
Signed-off-by: Jan Beulich <jbeulich@suse.com>
Tested-by: Dario Faggioli <dario.faggioli@citrix.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
Tested-by: Joao Martins <joao.m.martins@oracle.com>
--- a/xen/arch/x86/apic.c
+++ b/xen/arch/x86/apic.c
@@ -1160,8 +1160,7 @@ static int __init calibrate_APIC_clock(v
/*
* We wrapped around just now. Let's start:
*/
- if (cpu_has_tsc)
- rdtscll(t1);
+ t1 = rdtsc_ordered();
tt1 = apic_read(APIC_TMCCT);
/*
@@ -1171,8 +1170,7 @@ static int __init calibrate_APIC_clock(v
wait_8254_wraparound();
tt2 = apic_read(APIC_TMCCT);
- if (cpu_has_tsc)
- rdtscll(t2);
+ t2 = rdtsc_ordered();
/*
* The APIC bus clock counter is 32 bits only, it
--- a/xen/arch/x86/cpu/amd.c
+++ b/xen/arch/x86/cpu/amd.c
@@ -464,6 +464,9 @@ static void __devinit init_amd(struct cp
wrmsr_amd_safe(0xc001100d, l, h & ~1);
}
+ /* MFENCE stops RDTSC speculation */
+ __set_bit(X86_FEATURE_MFENCE_RDTSC, c->x86_capability);
+
switch(c->x86)
{
case 0xf ... 0x17:
--- a/xen/arch/x86/delay.c
+++ b/xen/arch/x86/delay.c
@@ -21,10 +21,10 @@ void __udelay(unsigned long usecs)
unsigned long ticks = usecs * (cpu_khz / 1000);
unsigned long s, e;
- rdtscl(s);
+ s = rdtsc_ordered();
do
{
rep_nop();
- rdtscl(e);
+ e = rdtsc_ordered();
} while ((e-s) < ticks);
}
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -142,7 +142,7 @@ static void synchronize_tsc_master(unsig
for ( i = 1; i <= 5; i++ )
{
- rdtscll(tsc_value);
+ tsc_value = rdtsc_ordered();
wmb();
atomic_inc(&tsc_count);
while ( atomic_read(&tsc_count) != (i<<1) )
--- a/xen/arch/x86/time.c
+++ b/xen/arch/x86/time.c
@@ -260,10 +260,10 @@ static u64 init_pit_and_calibrate_tsc(vo
outb(CALIBRATE_LATCH & 0xff, PIT_CH2); /* LSB of count */
outb(CALIBRATE_LATCH >> 8, PIT_CH2); /* MSB of count */
- rdtscll(start);
+ start = rdtsc_ordered();
for ( count = 0; (inb(0x61) & 0x20) == 0; count++ )
continue;
- rdtscll(end);
+ end = rdtsc_ordered();
/* Error if the CTC doesn't behave itself. */
if ( count == 0 )
@@ -763,7 +763,7 @@ s_time_t get_s_time_fixed(u64 at_tsc)
if ( at_tsc )
tsc = at_tsc;
else
- rdtscll(tsc);
+ tsc = rdtsc_ordered();
delta = tsc - t->local_tsc_stamp;
now = t->stime_local_stamp + scale_delta(delta, &t->tsc_scale);
@@ -958,7 +958,7 @@ int cpu_frequency_change(u64 freq)
/* TSC-extrapolated time may be bogus after frequency change. */
/*t->stime_local_stamp = get_s_time();*/
t->stime_local_stamp = t->stime_master_stamp;
- rdtscll(curr_tsc);
+ curr_tsc = rdtsc_ordered();
t->local_tsc_stamp = curr_tsc;
set_time_scale(&t->tsc_scale, freq);
local_irq_enable();
@@ -1170,16 +1170,13 @@ static void local_time_calibration(void)
*/
static void check_tsc_warp(unsigned long tsc_khz, unsigned long *max_warp)
{
-#define rdtsc_barrier() mb()
static DEFINE_SPINLOCK(sync_lock);
static cycles_t last_tsc;
cycles_t start, now, prev, end;
int i;
- rdtsc_barrier();
- start = get_cycles();
- rdtsc_barrier();
+ start = rdtsc_ordered();
/* The measurement runs for 20 msecs: */
end = start + tsc_khz * 20ULL;
@@ -1194,9 +1191,7 @@ static void check_tsc_warp(unsigned long
*/
spin_lock(&sync_lock);
prev = last_tsc;
- rdtsc_barrier();
- now = get_cycles();
- rdtsc_barrier();
+ now = rdtsc_ordered();
last_tsc = now;
spin_unlock(&sync_lock);
@@ -1294,7 +1289,7 @@ static void time_calibration_tsc_rendezv
if ( r->master_stime == 0 )
{
r->master_stime = read_platform_stime();
- rdtscll(r->master_tsc_stamp);
+ r->master_tsc_stamp = rdtsc_ordered();
}
atomic_inc(&r->semaphore);
@@ -1320,7 +1315,7 @@ static void time_calibration_tsc_rendezv
}
}
- rdtscll(c->local_tsc_stamp);
+ c->local_tsc_stamp = rdtsc_ordered();
c->stime_local_stamp = get_s_time_fixed(c->local_tsc_stamp);
c->stime_master_stamp = r->master_stime;
@@ -1350,7 +1345,7 @@ static void time_calibration_std_rendezv
mb(); /* receive signal /then/ read r->master_stime */
}
- rdtscll(c->local_tsc_stamp);
+ c->local_tsc_stamp = rdtsc_ordered();
c->stime_local_stamp = get_s_time_fixed(c->local_tsc_stamp);
c->stime_master_stamp = r->master_stime;
@@ -1385,7 +1380,7 @@ void time_latch_stamps(void)
local_irq_save(flags);
ap_bringup_ref.master_stime = read_platform_stime();
- rdtscll(tsc);
+ tsc = rdtsc_ordered();
local_irq_restore(flags);
ap_bringup_ref.local_stime = get_s_time_fixed(tsc);
@@ -1403,7 +1398,7 @@ void init_percpu_time(void)
local_irq_save(flags);
now = read_platform_stime();
- rdtscll(tsc);
+ tsc = rdtsc_ordered();
local_irq_restore(flags);
t->stime_master_stamp = now;
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -63,6 +63,7 @@
/* Other features, Linux-defined mapping, word 3 */
/* This range is used for feature bits which conflict or are synthesized */
+#define X86_FEATURE_MFENCE_RDTSC (3*32+ 7) /* MFENCE synchronizes RDTSC */
#define X86_FEATURE_CONSTANT_TSC (3*32+ 8) /* TSC ticks at a constant rate */
#define X86_FEATURE_NONSTOP_TSC (3*32+ 9) /* TSC does not stop in C states */
#define X86_FEATURE_ARAT (3*32+ 10) /* Always running APIC timer */
--- a/xen/include/asm-x86/msr.h
+++ b/xen/include/asm-x86/msr.h
@@ -83,6 +83,26 @@ static inline int wrmsr_safe(unsigned in
(val) = ((unsigned long)_eax) | (((unsigned long)_edx)<<32); \
} while(0)
+static inline uint64_t rdtsc_ordered(void)
+{
+ uint64_t tsc;
+
+ /*
+ * The RDTSC instruction is not ordered relative to memory access.
+ * The Intel SDM and the AMD APM are both vague on this point, but
+ * empirically an RDTSC instruction can be speculatively executed
+ * before prior loads. An RDTSC immediately after an appropriate
+ * barrier appears to be ordered as a normal load, that is, it
+ * provides the same ordering guarantees as reading from a global
+ * memory location that some other imaginary CPU is updating
+ * continuously with a time stamp.
+ */
+ alternative("lfence", "mfence", X86_FEATURE_MFENCE_RDTSC);
+ rdtscll(tsc);
+
+ return tsc;
+}
+
#define __write_tsc(val) wrmsrl(MSR_IA32_TSC, val)
#define write_tsc(val) ({ \
/* Reliable TSCs are in lockstep across all CPUs. We should \
