percpu: make this_cpu_generic_read() atomic w.r.t. interrupts

As raw_cpu_generic_read() is a plain read from a raw_cpu_ptr() address,
it's possible (albeit unlikely) that the compiler will split the access
across multiple instructions.

In this_cpu_generic_read() we disable preemption but not interrupts
before calling raw_cpu_generic_read(). Thus, an interrupt could be taken
in the middle of the split load instructions. If a this_cpu_write() or
RMW this_cpu_*() op is made to the same variable in the interrupt
handling path, this_cpu_read() will return a torn value.

For native word types, we can avoid tearing using READ_ONCE(), but this
won't work in all cases (e.g. 64-bit types on most 32-bit platforms).
This patch reworks this_cpu_generic_read() to use READ_ONCE() where
possible, otherwise falling back to disabling interrupts.

Signed-off-by: Mark Rutland <[email protected]>
Cc: Arnd Bergmann <[email protected]>
Cc: Christoph Lameter <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Pranith Kumar <[email protected]>
Cc: Tejun Heo <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Tejun Heo <[email protected]>

diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h
index 0504ef8f3aa31d5e7a9f0d86d1cb3fc130fee9e0..976f8ac26665b3f4f03cbbfe5222277d9827d847 100644 (file)
--- a/include/asm-generic/percpu.h
+++ b/include/asm-generic/percpu.h
@@ -115,15 +115,35 @@ do {                                                                      \
        (__ret);                                                        \
 })
 
-#define this_cpu_generic_read(pcp)                                     \
+#define __this_cpu_generic_read_nopreempt(pcp)                         \
 ({                                                                     \
        typeof(pcp) __ret;                                              \
        preempt_disable_notrace();                                      \
-       __ret = raw_cpu_generic_read(pcp);                              \
+       __ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));                        \
        preempt_enable_notrace();                                       \
        __ret;                                                          \
 })
 
+#define __this_cpu_generic_read_noirq(pcp)                             \
+({                                                                     \
+       typeof(pcp) __ret;                                              \
+       unsigned long __flags;                                          \
+       raw_local_irq_save(__flags);                                    \
+       __ret = raw_cpu_generic_read(pcp);                              \
+       raw_local_irq_restore(__flags);                                 \
+       __ret;                                                          \
+})
+
+#define this_cpu_generic_read(pcp)                                     \
+({                                                                     \
+       typeof(pcp) __ret;                                              \
+       if (__native_word(pcp))                                         \
+               __ret = __this_cpu_generic_read_nopreempt(pcp);         \
+       else                                                            \
+               __ret = __this_cpu_generic_read_noirq(pcp);             \
+       __ret;                                                          \
+})
+
 #define this_cpu_generic_to_op(pcp, val, op)                           \
 do {                                                                   \
        unsigned long __flags;                                          \