jump_label: Add RELEASE barrier after text changes

In the unlikely case text modification does not fully order things,
add some extra ordering of our own to ensure we only enabled the fast
path after all text is visible.

Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Cc: Jason Baron <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Paolo Bonzini <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Signed-off-by: Ingo Molnar <[email protected]>

diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 833eecae825ecff6bd9f988af7bac29a3a8ba981..f2ea678c269f5aeb445be04908746757716ba6bd 100644 (file)
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -108,7 +108,11 @@ void static_key_slow_inc(struct static_key *key)
        if (atomic_read(&key->enabled) == 0) {
                atomic_set(&key->enabled, -1);
                jump_label_update(key);
-               atomic_set(&key->enabled, 1);
+               /*
+                * Ensure that if the above cmpxchg loop observes our positive
+                * value, it must also observe all the text changes.
+                */
+               atomic_set_release(&key->enabled, 1);
        } else {
                atomic_inc(&key->enabled);
        }
@@ -130,7 +134,10 @@ void static_key_enable(struct static_key *key)
        if (atomic_read(&key->enabled) == 0) {
                atomic_set(&key->enabled, -1);
                jump_label_update(key);
-               atomic_set(&key->enabled, 1);
+               /*
+                * See static_key_slow_inc().
+                */
+               atomic_set_release(&key->enabled, 1);
        }
        jump_label_unlock();
        cpus_read_unlock();