percpu-rwsem: use synchronize_sched_expedited

Use synchronize_sched_expedited() instead of synchronize_sched()
to improve mount speed.

This patch improves mount time from 0.500s to 0.013s for Jeff's
test-case.

Signed-off-by: Mikulas Patocka <[email protected]>
Reported-and-tested-by: Jeff Chua <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
index 250a4acddb2b8975a6c8278a3c5b1d803f36b585..bd1e86071e57c27219b2746ab715d419c60fcc2c 100644 (file)
--- a/include/linux/percpu-rwsem.h
+++ b/include/linux/percpu-rwsem.h
@@ -13,7 +13,7 @@ struct percpu_rw_semaphore {
 };
 
 #define light_mb()     barrier()
-#define heavy_mb()     synchronize_sched()
+#define heavy_mb()     synchronize_sched_expedited()
 
 static inline void percpu_down_read(struct percpu_rw_semaphore *p)
 {
@@ -51,7 +51,7 @@ static inline void percpu_down_write(struct percpu_rw_semaphore *p)
 {
        mutex_lock(&p->mtx);
        p->locked = true;
-       synchronize_sched(); /* make sure that all readers exit the rcu_read_lock_sched region */
+       synchronize_sched_expedited(); /* make sure that all readers exit the rcu_read_lock_sched region */
        while (__percpu_count(p->counters))
                msleep(1);
        heavy_mb(); /* C, between read of p->counter and write to data, paired with B */