io_uring: move read/write file prep state into actual opcode handler

commit 584b0180f0f4d67d7145950fe68c625f06c88b10 upstream.

In preparation for not necessarily having a file assigned at prep time,
defer any initialization associated with the file to when the opcode
handler is run.

Cc: stable@vger.kernel.org # v5.15+
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index bb7edd1e3ffd31f85c0f5881d516c52a0248b1eb..c75e4a8ececa6a1571d2d0f2c5fbcba5e51169b4 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -560,7 +560,8 @@ struct io_rw {
        /* NOTE: kiocb has the file as the first member, so don't do it here */
        struct kiocb                    kiocb;
        u64                             addr;
-       u64                             len;
+       u32                             len;
+       u32                             flags;
 };
 
 struct io_connect {
@@ -2984,50 +2985,11 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req)
 
 static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
-       struct io_ring_ctx *ctx = req->ctx;
        struct kiocb *kiocb = &req->rw.kiocb;
-       struct file *file = req->file;
        unsigned ioprio;
        int ret;
 
-       if (!io_req_ffs_set(req))
-               req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;
-
        kiocb->ki_pos = READ_ONCE(sqe->off);
-       if (kiocb->ki_pos == -1) {
-               if (!(file->f_mode & FMODE_STREAM)) {
-                       req->flags |= REQ_F_CUR_POS;
-                       kiocb->ki_pos = file->f_pos;
-               } else {
-                       kiocb->ki_pos = 0;
-               }
-       }
-       kiocb->ki_flags = iocb_flags(file);
-       ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
-       if (unlikely(ret))
-               return ret;
-
-       /*
-        * If the file is marked O_NONBLOCK, still allow retry for it if it
-        * supports async. Otherwise it's impossible to use O_NONBLOCK files
-        * reliably. If not, or it IOCB_NOWAIT is set, don't retry.
-        */
-       if ((kiocb->ki_flags & IOCB_NOWAIT) ||
-           ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))
-               req->flags |= REQ_F_NOWAIT;
-
-       if (ctx->flags & IORING_SETUP_IOPOLL) {
-               if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)
-                       return -EOPNOTSUPP;
-
-               kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;
-               kiocb->ki_complete = io_complete_rw_iopoll;
-               req->iopoll_completed = 0;
-       } else {
-               if (kiocb->ki_flags & IOCB_HIPRI)
-                       return -EINVAL;
-               kiocb->ki_complete = io_complete_rw;
-       }
 
        ioprio = READ_ONCE(sqe->ioprio);
        if (ioprio) {
@@ -3043,6 +3005,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        req->imu = NULL;
        req->rw.addr = READ_ONCE(sqe->addr);
        req->rw.len = READ_ONCE(sqe->len);
+       req->rw.flags = READ_ONCE(sqe->rw_flags);
        req->buf_index = READ_ONCE(sqe->buf_index);
        return 0;
 }
@@ -3523,13 +3486,6 @@ static inline int io_rw_prep_async(struct io_kiocb *req, int rw)
        return 0;
 }
 
-static int io_read_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
-{
-       if (unlikely(!(req->file->f_mode & FMODE_READ)))
-               return -EBADF;
-       return io_prep_rw(req, sqe);
-}
-
 /*
  * This is our waitqueue callback handler, registered through __folio_lock_async()
  * when we initially tried to do the IO with the iocb armed our waitqueue.
@@ -3617,6 +3573,58 @@ static bool need_read_all(struct io_kiocb *req)
                S_ISBLK(file_inode(req->file)->i_mode);
 }
 
+static int io_rw_init_file(struct io_kiocb *req, fmode_t mode)
+{
+       struct kiocb *kiocb = &req->rw.kiocb;
+       struct io_ring_ctx *ctx = req->ctx;
+       struct file *file = req->file;
+       int ret;
+
+       if (unlikely(!file || !(file->f_mode & mode)))
+               return -EBADF;
+
+       if (!io_req_ffs_set(req))
+               req->flags |= io_file_get_flags(file) << REQ_F_SUPPORT_NOWAIT_BIT;
+
+       if (kiocb->ki_pos == -1) {
+               if (!(file->f_mode & FMODE_STREAM)) {
+                       req->flags |= REQ_F_CUR_POS;
+                       kiocb->ki_pos = file->f_pos;
+               } else {
+                       kiocb->ki_pos = 0;
+               }
+       }
+
+       kiocb->ki_flags = iocb_flags(file);
+       ret = kiocb_set_rw_flags(kiocb, req->rw.flags);
+       if (unlikely(ret))
+               return ret;
+
+       /*
+        * If the file is marked O_NONBLOCK, still allow retry for it if it
+        * supports async. Otherwise it's impossible to use O_NONBLOCK files
+        * reliably. If not, or it IOCB_NOWAIT is set, don't retry.
+        */
+       if ((kiocb->ki_flags & IOCB_NOWAIT) ||
+           ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req)))
+               req->flags |= REQ_F_NOWAIT;
+
+       if (ctx->flags & IORING_SETUP_IOPOLL) {
+               if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll)
+                       return -EOPNOTSUPP;
+
+               kiocb->ki_flags |= IOCB_HIPRI | IOCB_ALLOC_CACHE;
+               kiocb->ki_complete = io_complete_rw_iopoll;
+               req->iopoll_completed = 0;
+       } else {
+               if (kiocb->ki_flags & IOCB_HIPRI)
+                       return -EINVAL;
+               kiocb->ki_complete = io_complete_rw;
+       }
+
+       return 0;
+}
+
 static int io_read(struct io_kiocb *req, unsigned int issue_flags)
 {
        struct io_rw_state __s, *s = &__s;
@@ -3641,6 +3649,9 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags)
                iov_iter_restore(&s->iter, &s->iter_state);
                iovec = NULL;
        }
+       ret = io_rw_init_file(req, FMODE_READ);
+       if (unlikely(ret))
+               return ret;
        req->result = iov_iter_count(&s->iter);
 
        if (force_nonblock) {
@@ -3739,14 +3750,6 @@ out_free:
        return 0;
 }
 
-static int io_write_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
-{
-       if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
-               return -EBADF;
-       req->rw.kiocb.ki_hint = ki_hint_validate(file_write_hint(req->file));
-       return io_prep_rw(req, sqe);
-}
-
 static int io_write(struct io_kiocb *req, unsigned int issue_flags)
 {
        struct io_rw_state __s, *s = &__s;
@@ -3766,6 +3769,9 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags)
                iov_iter_restore(&s->iter, &s->iter_state);
                iovec = NULL;
        }
+       ret = io_rw_init_file(req, FMODE_WRITE);
+       if (unlikely(ret))
+               return ret;
        req->result = iov_iter_count(&s->iter);
 
        if (force_nonblock) {
@@ -6501,11 +6507,10 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
        case IORING_OP_READV:
        case IORING_OP_READ_FIXED:
        case IORING_OP_READ:
-               return io_read_prep(req, sqe);
        case IORING_OP_WRITEV:
        case IORING_OP_WRITE_FIXED:
        case IORING_OP_WRITE:
-               return io_write_prep(req, sqe);
+               return io_prep_rw(req, sqe);
        case IORING_OP_POLL_ADD:
                return io_poll_add_prep(req, sqe);
        case IORING_OP_POLL_REMOVE: