diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 0000913f7efc..310de0463beb 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -342,23 +342,19 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, { struct iov_iter iter; struct req_iterator rq_iter; - struct bio_vec *bvec; struct request *rq = blk_mq_rq_from_pdu(cmd); - struct bio *bio = rq->bio; struct file *file = lo->lo_backing_file; - struct bio_vec tmp; - unsigned int offset; unsigned int nr_bvec; int ret; nr_bvec = blk_rq_nr_bvec(rq); if (rq->bio != rq->biotail) { + struct bio_vec tmp, *bvec; - bvec = kmalloc_objs(struct bio_vec, nr_bvec, GFP_NOIO); - if (!bvec) + cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO); + if (!cmd->bvec) return -EIO; - cmd->bvec = bvec; /* * The bios of the request may be started from the middle of @@ -366,26 +362,26 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec * API will take care of all details for us. */ + bvec = cmd->bvec; rq_for_each_bvec(tmp, rq, rq_iter) { *bvec = tmp; bvec++; } - bvec = cmd->bvec; - offset = 0; + iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq)); + iter.iov_offset = 0; } else { /* * Same here, this bio may be started from the middle of the * 'bvec' because of bio splitting, so offset from the bvec * must be passed to iov iterator */ - offset = bio->bi_iter.bi_bvec_done; - bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); + iov_iter_bvec(&iter, rw, + __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), + nr_bvec, blk_rq_bytes(rq)); + iter.iov_offset = rq->bio->bi_iter.bi_bvec_done; } atomic_set(&cmd->ref, 2); - iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq)); - iter.iov_offset = offset; - cmd->iocb.ki_pos = pos; cmd->iocb.ki_filp = file; cmd->iocb.ki_ioprio = req_get_ioprio(rq); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 15d36d6a728e..9449190acc5a 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -340,32 +340,25 @@ static void nvme_tcp_init_iter(struct nvme_tcp_request *req, unsigned int dir) { struct request *rq = blk_mq_rq_from_pdu(req); - struct bio_vec *vec; - unsigned int size; - int nr_bvec; - size_t offset; if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { - vec = &rq->special_vec; - nr_bvec = 1; - size = blk_rq_payload_bytes(rq); - offset = 0; + iov_iter_bvec(&req->iter, dir, &rq->special_vec, 1, + blk_rq_payload_bytes(rq)); + req->iter.iov_offset = 0; } else { struct bio *bio = req->curr_bio; struct bvec_iter bi; struct bio_vec bv; + int nr_bvec = 0; - vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); - nr_bvec = 0; - bio_for_each_bvec(bv, bio, bi) { + bio_for_each_bvec(bv, bio, bi) nr_bvec++; - } - size = bio->bi_iter.bi_size; - offset = bio->bi_iter.bi_bvec_done; - } - iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); - req->iter.iov_offset = offset; + iov_iter_bvec(&req->iter, dir, + __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter), nr_bvec, + bio->bi_iter.bi_size); + req->iter.iov_offset = bio->bi_iter.bi_bvec_done; + } } static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, diff --git a/include/linux/bvec.h b/include/linux/bvec.h index d36dd476feda..f4c7ec282ac9 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -104,51 +104,78 @@ struct bvec_iter_all { unsigned done; }; -/* - * various member access, note that bio_data should of course not be used - * on highmem page vectors - */ -#define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) +static __always_inline const struct bio_vec * +__bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return bvecs + iter.bi_idx; +} /* multi-page (mp_bvec) helpers */ -#define mp_bvec_iter_page(bvec, iter) \ - (__bvec_iter_bvec((bvec), (iter))->bv_page) +static __always_inline struct page * +mp_bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return __bvec_iter_bvec(bvecs, iter)->bv_page; +} -#define mp_bvec_iter_len(bvec, iter) \ - min((iter).bi_size, \ - __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) +static __always_inline unsigned int +mp_bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return min(__bvec_iter_bvec(bvecs, iter)->bv_len - iter.bi_bvec_done, + iter.bi_size); +} -#define mp_bvec_iter_offset(bvec, iter) \ - (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) +static __always_inline unsigned int +mp_bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return __bvec_iter_bvec(bvecs, iter)->bv_offset + iter.bi_bvec_done; +} -#define mp_bvec_iter_page_idx(bvec, iter) \ - (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE) +static __always_inline unsigned int +mp_bvec_iter_page_idx(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_offset(bvecs, iter) / PAGE_SIZE; +} -#define mp_bvec_iter_bvec(bvec, iter) \ -((struct bio_vec) { \ - .bv_page = mp_bvec_iter_page((bvec), (iter)), \ - .bv_len = mp_bvec_iter_len((bvec), (iter)), \ - .bv_offset = mp_bvec_iter_offset((bvec), (iter)), \ -}) +static __always_inline struct bio_vec +mp_bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return (struct bio_vec) { + .bv_page = mp_bvec_iter_page(bvecs, iter), + .bv_len = mp_bvec_iter_len(bvecs, iter), + .bv_offset = mp_bvec_iter_offset(bvecs, iter), + }; +} /* For building single-page bvec in flight */ - #define bvec_iter_offset(bvec, iter) \ - (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE) - -#define bvec_iter_len(bvec, iter) \ - min_t(unsigned, mp_bvec_iter_len((bvec), (iter)), \ - PAGE_SIZE - bvec_iter_offset((bvec), (iter))) - -#define bvec_iter_page(bvec, iter) \ - (mp_bvec_iter_page((bvec), (iter)) + \ - mp_bvec_iter_page_idx((bvec), (iter))) - -#define bvec_iter_bvec(bvec, iter) \ -((struct bio_vec) { \ - .bv_page = bvec_iter_page((bvec), (iter)), \ - .bv_len = bvec_iter_len((bvec), (iter)), \ - .bv_offset = bvec_iter_offset((bvec), (iter)), \ -}) +static __always_inline unsigned int +bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_offset(bvecs, iter) % PAGE_SIZE; +} + +static __always_inline unsigned int +bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return min(mp_bvec_iter_len(bvecs, iter), + PAGE_SIZE - bvec_iter_offset(bvecs, iter)); +} + +static __always_inline struct page * +bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_page(bvecs, iter) + + mp_bvec_iter_page_idx(bvecs, iter); +} + +static __always_inline struct bio_vec +bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return (struct bio_vec) { + .bv_page = bvec_iter_page(bvecs, iter), + .bv_len = bvec_iter_len(bvecs, iter), + .bv_offset = bvec_iter_offset(bvecs, iter), + }; +} static inline bool bvec_iter_advance(const struct bio_vec *bv, struct bvec_iter *iter, unsigned bytes)