rcu: Make TASKS_RCU handle nohz_full= CPUs

Currently TASKS_RCU would ignore a CPU running a task in nohz_full=
usermode execution.  There would be neither a context switch nor a
scheduling-clock interrupt to tell TASKS_RCU that the task in question
had passed through a quiescent state.  The grace period would therefore
extend indefinitely.  This commit therefore makes RCU's dyntick-idle
subsystem record the task_struct structure of the task that is running
in dyntick-idle mode on each CPU.  The TASKS_RCU grace period can
then access this information and record a quiescent state on
behalf of any CPU running in dyntick-idle usermode.

Signed-off-by: Paul E. McKenney <[email protected]>

diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index dffd9258ee60b61c25351c6b1d2111fe63c81d09..03b274873b06f761da96def05383c6bbebe0aa41 100644 (file)
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -121,7 +121,8 @@ extern struct group_info init_groups;
 #define INIT_TASK_RCU_TASKS(tsk)                                       \
        .rcu_tasks_holdout = false,                                     \
        .rcu_tasks_holdout_list =                                       \
-               LIST_HEAD_INIT(tsk.rcu_tasks_holdout_list),
+               LIST_HEAD_INIT(tsk.rcu_tasks_holdout_list),             \
+       .rcu_tasks_idle_cpu = -1,
 #else
 #define INIT_TASK_RCU_TASKS(tsk)
 #endif

diff --git a/include/linux/sched.h b/include/linux/sched.h
index eaacac4ae77d58eb3e6f5c237d0b3e1bd1e8f962..ec8b34722bccbb741ea5f91d09c9cc9adc24d12b 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1274,6 +1274,7 @@ struct task_struct {
        unsigned long rcu_tasks_nvcsw;
        bool rcu_tasks_holdout;
        struct list_head rcu_tasks_holdout_list;
+       int rcu_tasks_idle_cpu;
 #endif /* #ifdef CONFIG_TASKS_RCU */
 
 #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
@@ -2020,6 +2021,7 @@ static inline void rcu_copy_process(struct task_struct *p)
 #ifdef CONFIG_TASKS_RCU
        p->rcu_tasks_holdout = false;
        INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
+       p->rcu_tasks_idle_cpu = -1;
 #endif /* #ifdef CONFIG_TASKS_RCU */
 }
 

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e23dad0661e231b479f733f83f09e26f6273df9c..c880f5387b1fd3e43d4bc1ae752218abdb4081d0 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -526,6 +526,7 @@ static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval,
        atomic_inc(&rdtp->dynticks);
        smp_mb__after_atomic();  /* Force ordering with next sojourn. */
        WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+       rcu_dynticks_task_enter();
 
        /*
         * It is illegal to enter an extended quiescent state while
@@ -642,6 +643,7 @@ void rcu_irq_exit(void)
 static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval,
                               int user)
 {
+       rcu_dynticks_task_exit();
        smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
        atomic_inc(&rdtp->dynticks);
        /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */

diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 6a86eb7bac45e7b540f1ab9ead7d412cf6ef60fa..3a92000c354fe94088cff5b8c92575f449ebc5f8 100644 (file)
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -605,6 +605,8 @@ static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
 static void rcu_bind_gp_kthread(void);
 static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp);
 static bool rcu_nohz_full_cpu(struct rcu_state *rsp);
+static void rcu_dynticks_task_enter(void);
+static void rcu_dynticks_task_exit(void);
 
 #endif /* #ifndef RCU_TREE_NONCORE */
 

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 7672586d3920fac44ee4123f1e087d2ce936f317..e466b40052a7069d6f5f3d6bcdc133f979e2347c 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -3036,3 +3036,19 @@ static void rcu_bind_gp_kthread(void)
                housekeeping_affine(current);
 #endif /* #else #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 }
+
+/* Record the current task on dyntick-idle entry. */
+static void rcu_dynticks_task_enter(void)
+{
+#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
+       ACCESS_ONCE(current->rcu_tasks_idle_cpu) = smp_processor_id();
+#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
+}
+
+/* Record no current task on dyntick-idle exit. */
+static void rcu_dynticks_task_exit(void)
+{
+#if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
+       ACCESS_ONCE(current->rcu_tasks_idle_cpu) = -1;
+#endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
+}

diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index e1d71741958f69febc8f4f8edce6effbf909f010..2658de4a5975967667d716eff1525d9e92b9a4c3 100644 (file)
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -463,7 +463,9 @@ static void check_holdout_task(struct task_struct *t,
 {
        if (!ACCESS_ONCE(t->rcu_tasks_holdout) ||
            t->rcu_tasks_nvcsw != ACCESS_ONCE(t->nvcsw) ||
-           !ACCESS_ONCE(t->on_rq)) {
+           !ACCESS_ONCE(t->on_rq) ||
+           (IS_ENABLED(CONFIG_NO_HZ_FULL) &&
+            !is_idle_task(t) && t->rcu_tasks_idle_cpu >= 0)) {
                ACCESS_ONCE(t->rcu_tasks_holdout) = false;
                list_del_rcu(&t->rcu_tasks_holdout_list);
                put_task_struct(t);