aio: give shared kioctx fields their own cachelines

[[email protected]: make reqs_active __cacheline_aligned_in_smp]
Signed-off-by: Kent Overstreet <[email protected]>
Cc: Zach Brown <[email protected]>
Cc: Felipe Balbi <[email protected]>
Cc: Greg Kroah-Hartman <[email protected]>
Cc: Mark Fasheh <[email protected]>
Cc: Joel Becker <[email protected]>
Cc: Rusty Russell <[email protected]>
Cc: Jens Axboe <[email protected]>
Cc: Asai Thambi S P <[email protected]>
Cc: Selvan Mani <[email protected]>
Cc: Sam Bradshaw <[email protected]>
Cc: Jeff Moyer <[email protected]>
Cc: Al Viro <[email protected]>
Cc: Benjamin LaHaise <[email protected]>
Reviewed-by: "Theodore Ts'o" <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>

diff --git a/fs/aio.c b/fs/aio.c
index 16239fbd64589dc2ac2536ea6fcab64018e80afe..986ff305a856c87e5e5299e07cc0bb24b67c1ad6 100644 (file)
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -67,13 +67,6 @@ struct kioctx {
        unsigned long           user_id;
        struct hlist_node       list;
 
-       wait_queue_head_t       wait;
-
-       spinlock_t              ctx_lock;
-
-       atomic_t                reqs_active;
-       struct list_head        active_reqs;    /* used for cancellation */
-
        /*
         * This is what userspace passed to io_setup(), it's not used for
         * anything but counting against the global max_reqs quota.
@@ -92,19 +85,29 @@ struct kioctx {
        struct page             **ring_pages;
        long                    nr_pages;
 
+       struct rcu_head         rcu_head;
+       struct work_struct      rcu_work;
+
+       struct {
+               atomic_t        reqs_active;
+       } ____cacheline_aligned_in_smp;
+
+       struct {
+               spinlock_t      ctx_lock;
+               struct list_head active_reqs;   /* used for cancellation */
+       } ____cacheline_aligned_in_smp;
+
        struct {
                struct mutex    ring_lock;
-       } ____cacheline_aligned;
+               wait_queue_head_t wait;
+       } ____cacheline_aligned_in_smp;
 
        struct {
                unsigned        tail;
                spinlock_t      completion_lock;
-       } ____cacheline_aligned;
+       } ____cacheline_aligned_in_smp;
 
        struct page             *internal_pages[AIO_RING_PAGES];
-
-       struct rcu_head         rcu_head;
-       struct work_struct      rcu_work;
 };
 
 /*------ sysctl variables----*/