blk: make the bioset rescue_workqueue optional.

This patch converts bioset_create() to not create a workqueue by
default, so alloctions will never trigger punt_bios_to_rescuer().  It
also introduces a new flag BIOSET_NEED_RESCUER which tells
bioset_create() to preserve the old behavior.

All callers of bioset_create() that are inside block device drivers,
are given the BIOSET_NEED_RESCUER flag.

biosets used by filesystems or other top-level users do not
need rescuing as the bio can never be queued behind other
bios.  This includes fs_bio_set, blkdev_dio_pool,
btrfs_bioset, xfs_ioend_bioset, and one allocated by
target_core_iblock.c.

biosets used by md/raid do not need rescuing as
their usage was recently audited and revised to never
risk deadlock.

It is hoped that most, if not all, of the remaining biosets
can end up being the non-rescued version.

Reviewed-by: Christoph Hellwig <[email protected]>
Credit-to: Ming Lei <[email protected]> (minor fixes)
Reviewed-by: Ming Lei <[email protected]>
Signed-off-by: NeilBrown <[email protected]>
Signed-off-by: Jens Axboe <[email protected]>

diff --git a/block/bio.c b/block/bio.c
index 84b313bd3ce824838486aec1bfff880bab8bbee2..2bd064906e068995cdc5163b795424c6fb30fff8 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -363,6 +363,8 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
        struct bio_list punt, nopunt;
        struct bio *bio;
 
+       if (WARN_ON_ONCE(!bs->rescue_workqueue))
+               return;
        /*
         * In order to guarantee forward progress we must punt only bios that
         * were allocated from this bio_set; otherwise, if there was a bio on
@@ -474,7 +476,8 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 
                if (current->bio_list &&
                    (!bio_list_empty(&current->bio_list[0]) ||
-                    !bio_list_empty(&current->bio_list[1])))
+                    !bio_list_empty(&current->bio_list[1])) &&
+                   bs->rescue_workqueue)
                        gfp_mask &= ~__GFP_DIRECT_RECLAIM;
 
                p = mempool_alloc(bs->bio_pool, gfp_mask);
@@ -1925,7 +1928,8 @@ EXPORT_SYMBOL(bioset_free);
  * bioset_create  - Create a bio_set
  * @pool_size: Number of bio and bio_vecs to cache in the mempool
  * @front_pad: Number of bytes to allocate in front of the returned bio
- * @flags:     Flags to modify behavior, currently only %BIOSET_NEED_BVECS
+ * @flags:     Flags to modify behavior, currently %BIOSET_NEED_BVECS
+ *              and %BIOSET_NEED_RESCUER
  *
  * Description:
  *    Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
@@ -1936,6 +1940,8 @@ EXPORT_SYMBOL(bioset_free);
  *    or things will break badly.
  *    If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
  *    for allocating iovecs.  This pool is not needed e.g. for bio_clone_fast().
+ *    If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to
+ *    dispatch queued requests when the mempool runs out of space.
  *
  */
 struct bio_set *bioset_create(unsigned int pool_size,
@@ -1971,6 +1977,9 @@ struct bio_set *bioset_create(unsigned int pool_size,
                        goto bad;
        }
 
+       if (!(flags & BIOSET_NEED_RESCUER))
+               return bs;
+
        bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
        if (!bs->rescue_workqueue)
                goto bad;

diff --git a/block/blk-core.c b/block/blk-core.c
index 62cf92550512241249c4bc4f05cae285e75587db..9bd10c46a538051da78011409eb725a8a1c94444 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -790,7 +790,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
        if (q->id < 0)
                goto fail_q;
 
-       q->bio_split = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
+       q->bio_split = bioset_create(BIO_POOL_SIZE, 0, (BIOSET_NEED_BVECS |
+                                                       BIOSET_NEED_RESCUER));
        if (!q->bio_split)
                goto fail_id;
 

diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index b395fe391171cdcd171be14add2f05697d4bd941..bdf51b6977cf58efe82aa19c4de73cfa92bb35ab 100644 (file)
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -2165,7 +2165,9 @@ static int drbd_create_mempools(void)
                goto Enomem;
 
        /* mempools */
-       drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0, BIOSET_NEED_BVECS);
+       drbd_md_io_bio_set = bioset_create(DRBD_MIN_POOL_PAGES, 0,
+                                          BIOSET_NEED_BVECS |
+                                          BIOSET_NEED_RESCUER);
        if (drbd_md_io_bio_set == NULL)
                goto Enomem;
 

diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index abd6e825b39be74e307167d9e2daa6a9c3d70508..8352fad765f61991c9725007ecbedff323ddacdb 100644 (file)
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -782,7 +782,9 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,
 
        minor *= BCACHE_MINORS;
 
-       if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio), BIOSET_NEED_BVECS)) ||
+       if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio),
+                                          BIOSET_NEED_BVECS |
+                                          BIOSET_NEED_RESCUER)) ||
            !(d->disk = alloc_disk(BCACHE_MINORS))) {
                ida_simple_remove(&bcache_minor, minor);
                return -ENOMEM;
@@ -1516,7 +1518,9 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
                                sizeof(struct bbio) + sizeof(struct bio_vec) *
                                bucket_pages(c))) ||
            !(c->fill_iter = mempool_create_kmalloc_pool(1, iter_size)) ||
-           !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio), BIOSET_NEED_BVECS)) ||
+           !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio),
+                                          BIOSET_NEED_BVECS |
+                                          BIOSET_NEED_RESCUER)) ||
            !(c->uuids = alloc_bucket_pages(GFP_KERNEL, c)) ||
            !(c->moving_gc_wq = alloc_workqueue("bcache_gc",
                                                WQ_MEM_RECLAIM, 0)) ||

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 237ff8e9752a2e01318e6d50057b2a286c62b82b..9e1b72e8f7efb5b62f97396d14554ecd7b39289f 100644 (file)
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -2677,7 +2677,8 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
                goto bad;
        }
 
-       cc->bs = bioset_create(MIN_IOS, 0, BIOSET_NEED_BVECS);
+       cc->bs = bioset_create(MIN_IOS, 0, (BIOSET_NEED_BVECS |
+                                           BIOSET_NEED_RESCUER));
        if (!cc->bs) {
                ti->error = "Cannot allocate crypt bioset";
                goto bad;

diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index 5c4121024d92ef37ac49f978ab8dc20950c5242f..81248a8a8b5706dfe6a03d4cbab3aa0b5910f2d4 100644 (file)
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -58,7 +58,8 @@ struct dm_io_client *dm_io_client_create(void)
        if (!client->pool)
                goto bad;
 
-       client->bios = bioset_create(min_ios, 0, BIOSET_NEED_BVECS);
+       client->bios = bioset_create(min_ios, 0, (BIOSET_NEED_BVECS |
+                                                 BIOSET_NEED_RESCUER));
        if (!client->bios)
                goto bad;
 

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 3394a311de3d0d051622fb29891aa500d0ff2bcf..fbd06b9f946748d34caface2ebbed308051b2ef6 100644 (file)
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1036,7 +1036,8 @@ static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule)
 
                while ((bio = bio_list_pop(&list))) {
                        struct bio_set *bs = bio->bi_pool;
-                       if (unlikely(!bs) || bs == fs_bio_set) {
+                       if (unlikely(!bs) || bs == fs_bio_set ||
+                           !bs->rescue_workqueue) {
                                bio_list_add(&current->bio_list[i], bio);
                                continue;
                        }
@@ -2660,7 +2661,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
                BUG();
        }
 
-       pools->bs = bioset_create(pool_size, front_pad, 0);
+       pools->bs = bioset_create(pool_size, front_pad, BIOSET_NEED_RESCUER);
        if (!pools->bs)
                goto out;
 

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 985dc645637ecbed2a6fcc0dabf8d26cd091c345..32c786baa10ab4e6f8572c988e6e8b7921b36f7b 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -376,6 +376,7 @@ static inline struct bio *bio_next_split(struct bio *bio, int sectors,
 extern struct bio_set *bioset_create(unsigned int, unsigned int, int flags);
 enum {
        BIOSET_NEED_BVECS = BIT(0),
+       BIOSET_NEED_RESCUER = BIT(1),
 };
 extern void bioset_free(struct bio_set *);
 extern mempool_t *biovec_create_pool(int pool_entries);