sched: Exclude cond_resched() from nested sleep test

cond_resched() is a preemption point, not strictly a blocking
primitive, so exclude it from the ->state test.

In particular, preemption preserves task_struct::state.

Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: Alex Elder <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Axel Lin <[email protected]>
Cc: Daniel Borkmann <[email protected]>
Cc: Dave Jones <[email protected]>
Cc: Jason Baron <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Rusty Russell <[email protected]>
Cc: Steven Rostedt <[email protected]>
Link: http://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>

diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index 5068a0d9fecd76a901921324420c4656b2b53c4d..446d76a87ba17f8b3116482a8617df1c5972bc1a 100644 (file)
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -162,6 +162,7 @@ extern int _cond_resched(void);
 #endif
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+  void ___might_sleep(const char *file, int line, int preempt_offset);
   void __might_sleep(const char *file, int line, int preempt_offset);
 /**
  * might_sleep - annotation for functions that can sleep
@@ -177,6 +178,8 @@ extern int _cond_resched(void);
        do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0)
 # define sched_annotate_sleep()        __set_current_state(TASK_RUNNING)
 #else
+  static inline void ___might_sleep(const char *file, int line,
+                                  int preempt_offset) { }
   static inline void __might_sleep(const char *file, int line,
                                   int preempt_offset) { }
 # define might_sleep() do { might_resched(); } while (0)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 4648e07f7d6f9f893868ee1effd6989d73c8002e..4400ddc2fe73c98406eecec1207f1a45d10cc4cf 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2806,7 +2806,7 @@ static inline int signal_pending_state(long state, struct task_struct *p)
 extern int _cond_resched(void);
 
 #define cond_resched() ({                      \
-       __might_sleep(__FILE__, __LINE__, 0);   \
+       ___might_sleep(__FILE__, __LINE__, 0);  \
        _cond_resched();                        \
 })
 
@@ -2819,14 +2819,14 @@ extern int __cond_resched_lock(spinlock_t *lock);
 #endif
 
 #define cond_resched_lock(lock) ({                             \
-       __might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET); \
+       ___might_sleep(__FILE__, __LINE__, PREEMPT_LOCK_OFFSET);\
        __cond_resched_lock(lock);                              \
 })
 
 extern int __cond_resched_softirq(void);
 
 #define cond_resched_softirq() ({                                      \
-       __might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET);      \
+       ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET);     \
        __cond_resched_softirq();                                       \
 })
 

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5b4b96b27cd7df2411c9c896356b21b9d0600f58..b9f78f12ac22ec706878d9965eff6d132c778f02 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -7296,8 +7296,6 @@ static inline int preempt_count_equals(int preempt_offset)
 
 void __might_sleep(const char *file, int line, int preempt_offset)
 {
-       static unsigned long prev_jiffy;        /* ratelimiting */
-
        /*
         * Blocking primitives will set (and therefore destroy) current->state,
         * since we will exit with TASK_RUNNING make sure we enter with it,
@@ -7311,6 +7309,14 @@ void __might_sleep(const char *file, int line, int preempt_offset)
                        (void *)current->task_state_change))
                __set_current_state(TASK_RUNNING);
 
+       ___might_sleep(file, line, preempt_offset);
+}
+EXPORT_SYMBOL(__might_sleep);
+
+void ___might_sleep(const char *file, int line, int preempt_offset)
+{
+       static unsigned long prev_jiffy;        /* ratelimiting */
+
        rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
        if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
             !is_idle_task(current)) ||
@@ -7340,7 +7346,7 @@ void __might_sleep(const char *file, int line, int preempt_offset)
 #endif
        dump_stack();
 }
-EXPORT_SYMBOL(__might_sleep);
+EXPORT_SYMBOL(___might_sleep);
 #endif
 
 #ifdef CONFIG_MAGIC_SYSRQ