fs/fuse kio: invalidate files for kio

Submitted by Pavel Butsykin on Jan. 18, 2019, 1:43 p.m.

Details

Message ID 20190118134332.22414-1-pbutsykin@virtuozzo.com
State New
Series "fs/fuse kio: invalidate files for kio"
Headers show

Commit Message

Pavel Butsykin Jan. 18, 2019, 1:43 p.m.
Make fuse_invalidate_files() work for kio. This is necessary to maintain
vstorage revoke in FPath mode. To do this, let's add a list of inflight kio
requests to be able to handle this list in fuse_invalidate_files(). The list
will also be useful to implement the full-featured .vstorage.info and
fuse_abort_conn() for kio.

#VSTOR-19620

Signed-off-by: Pavel Butsykin <pbutsykin@virtuozzo.com>
---
 fs/fuse/fuse_i.h                   |  4 ++++
 fs/fuse/inode.c                    |  8 ++++++--
 fs/fuse/kio/pcs/fuse_io.c          |  5 +++++
 fs/fuse/kio/pcs/pcs_cluster.c      |  2 ++
 fs/fuse/kio/pcs/pcs_cluster.h      |  1 +
 fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 16 ++++++++++++++++
 6 files changed, 34 insertions(+), 2 deletions(-)

Patch hide | download patch | download mbox

diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index 4c27bfa0e74d..0914940d6735 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -528,6 +528,7 @@  struct fuse_kio_ops {
 	int  (*file_open)(struct fuse_conn *fc, struct file *file,
 			  struct inode *inode);
 	void (*inode_release)(struct fuse_inode *fi);
+	void (*kill_requests)(struct fuse_conn *fc, struct inode *inode);
 
 };
 int fuse_register_kio(struct fuse_kio_ops *ops);
@@ -1107,4 +1108,7 @@  struct fuse_file *fuse_write_file(struct fuse_conn *fc, struct fuse_inode *fi);
 
 void fuse_release_ff(struct inode *inode, struct fuse_file *ff);
 
+void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
+			struct list_head *req_list);
+
 #endif /* _FS_FUSE_I_H */
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index 34e52262d37e..cb275ff21991 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -369,8 +369,8 @@  int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid,
 	return 0;
 }
 
-static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
-			       struct list_head *req_list)
+void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
+			struct list_head *req_list)
 {
 	struct fuse_req *req;
 
@@ -393,6 +393,7 @@  static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
 			req->num_pages = 0;
 		}
 }
+EXPORT_SYMBOL_GPL(fuse_kill_requests);
 
 int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
 {
@@ -442,6 +443,9 @@  int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
 		}
 		fuse_kill_requests(fc, inode, &fc->main_iq.pending);
 		fuse_kill_requests(fc, inode, &fc->bg_queue);
+		if (fc->kio.op && fc->kio.op->kill_requests)
+			fc->kio.op->kill_requests(fc, inode);
+
 		wake_up(&fi->page_waitq); /* readpage[s] can wait on fuse wb */
 		spin_unlock(&fc->lock);
 
diff --git a/fs/fuse/kio/pcs/fuse_io.c b/fs/fuse/kio/pcs/fuse_io.c
index 61a2e7f1eac6..00a72c878efb 100644
--- a/fs/fuse/kio/pcs/fuse_io.c
+++ b/fs/fuse/kio/pcs/fuse_io.c
@@ -205,6 +205,7 @@  static void prepare_io_(struct pcs_fuse_req *r, unsigned short type, off_t offse
 static void ioreq_complete(pcs_api_iorequest_t *ioreq)
 {
 	struct pcs_fuse_req *r = ioreq->datasource;
+	struct fuse_conn *fc = cl_from_req(r)->fc;
 
 	BUG_ON(ioreq != &r->exec.io.req);
 
@@ -217,6 +218,10 @@  static void ioreq_complete(pcs_api_iorequest_t *ioreq)
 		r->req.out.h.error = 0;
 	}
 
+	spin_lock(&fc->lock);
+	list_del_init(&r->req.list);
+	spin_unlock(&fc->lock);
+
 	switch (ioreq->type) {
 	case PCS_REQ_T_READ:
 		on_read_done(r, ioreq->size);
diff --git a/fs/fuse/kio/pcs/pcs_cluster.c b/fs/fuse/kio/pcs/pcs_cluster.c
index 5df263f01f98..6b8f38db5c40 100644
--- a/fs/fuse/kio/pcs/pcs_cluster.c
+++ b/fs/fuse/kio/pcs/pcs_cluster.c
@@ -608,6 +608,8 @@  int pcs_cluster_init(struct pcs_fuse_cluster *pfc, struct workqueue_struct *wq,
 	pfc->cc.op.ireq_on_error   = ireq_on_error_;
 	pfc->cc.op.ireq_check_redo = ireq_check_redo_;
 
+	INIT_LIST_HEAD(&pfc->kio_queue);
+
 	return 0;
 }
 
diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
index 9c537cb43b30..8a3ae3244b80 100644
--- a/fs/fuse/kio/pcs/pcs_cluster.h
+++ b/fs/fuse/kio/pcs/pcs_cluster.h
@@ -32,6 +32,7 @@  struct pcs_fuse_req {
 struct pcs_fuse_cluster {
 	struct pcs_cluster_core cc;
 	struct fuse_conn *fc;
+	struct list_head kio_queue;
 };
 
 struct pcs_fuse_work {
diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
index cdd4fe578128..1ca27b80b002 100644
--- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
+++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
@@ -983,6 +983,12 @@  error:
 	return;
 
 submit:
+	if (!lk)
+		spin_lock(&pfc->fc->lock);
+	list_add_tail(&req->list, &pfc->kio_queue);
+	if (!lk)
+		spin_unlock(&pfc->fc->lock);
+
 	if (async)
 		pcs_cc_submit(ireq->cc, ireq);
 	else
@@ -1488,6 +1494,15 @@  void __kfuse_trace(struct fuse_conn * fc, unsigned long ip, const char * fmt, ..
 	put_cpu();
 }
 
+static void kpcs_kill_requests(struct fuse_conn *fc, struct inode *inode)
+{
+	struct pcs_fuse_cluster *pfc = (struct pcs_fuse_cluster*)fc->kio.ctx;
+
+	assert_spin_locked(&fc->lock);
+
+	return fuse_kill_requests(fc, inode, &pfc->kio_queue);
+}
+
 static struct fuse_kio_ops kio_pcs_ops = {
 	.name		= "pcs",
 	.owner		= THIS_MODULE,
@@ -1500,6 +1515,7 @@  static struct fuse_kio_ops kio_pcs_ops = {
 	.req_send	= kpcs_req_send,
 	.file_open	= kpcs_file_open,
 	.inode_release	= kpcs_inode_release,
+	.kill_requests	= kpcs_kill_requests,
 };
 
 

Comments

Alexey Kuznetsov Jan. 18, 2019, 3 p.m.
Hello!

Huh? We were going to get rid of fc->lock and definitely are not going
to add new ones.
It is exactly why it was not made in the first place.

On Fri, Jan 18, 2019 at 4:44 PM Pavel Butsykin <pbutsykin@virtuozzo.com> wrote:
>
> Make fuse_invalidate_files() work for kio. This is necessary to maintain
> vstorage revoke in FPath mode. To do this, let's add a list of inflight kio
> requests to be able to handle this list in fuse_invalidate_files(). The list
> will also be useful to implement the full-featured .vstorage.info and
> fuse_abort_conn() for kio.
>
> #VSTOR-19620
>
> Signed-off-by: Pavel Butsykin <pbutsykin@virtuozzo.com>
> ---
>  fs/fuse/fuse_i.h                   |  4 ++++
>  fs/fuse/inode.c                    |  8 ++++++--
>  fs/fuse/kio/pcs/fuse_io.c          |  5 +++++
>  fs/fuse/kio/pcs/pcs_cluster.c      |  2 ++
>  fs/fuse/kio/pcs/pcs_cluster.h      |  1 +
>  fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 16 ++++++++++++++++
>  6 files changed, 34 insertions(+), 2 deletions(-)
>
> diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
> index 4c27bfa0e74d..0914940d6735 100644
> --- a/fs/fuse/fuse_i.h
> +++ b/fs/fuse/fuse_i.h
> @@ -528,6 +528,7 @@ struct fuse_kio_ops {
>         int  (*file_open)(struct fuse_conn *fc, struct file *file,
>                           struct inode *inode);
>         void (*inode_release)(struct fuse_inode *fi);
> +       void (*kill_requests)(struct fuse_conn *fc, struct inode *inode);
>
>  };
>  int fuse_register_kio(struct fuse_kio_ops *ops);
> @@ -1107,4 +1108,7 @@ struct fuse_file *fuse_write_file(struct fuse_conn *fc, struct fuse_inode *fi);
>
>  void fuse_release_ff(struct inode *inode, struct fuse_file *ff);
>
> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
> +                       struct list_head *req_list);
> +
>  #endif /* _FS_FUSE_I_H */
> diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
> index 34e52262d37e..cb275ff21991 100644
> --- a/fs/fuse/inode.c
> +++ b/fs/fuse/inode.c
> @@ -369,8 +369,8 @@ int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid,
>         return 0;
>  }
>
> -static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
> -                              struct list_head *req_list)
> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
> +                       struct list_head *req_list)
>  {
>         struct fuse_req *req;
>
> @@ -393,6 +393,7 @@ static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>                         req->num_pages = 0;
>                 }
>  }
> +EXPORT_SYMBOL_GPL(fuse_kill_requests);
>
>  int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>  {
> @@ -442,6 +443,9 @@ int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>                 }
>                 fuse_kill_requests(fc, inode, &fc->main_iq.pending);
>                 fuse_kill_requests(fc, inode, &fc->bg_queue);
> +               if (fc->kio.op && fc->kio.op->kill_requests)
> +                       fc->kio.op->kill_requests(fc, inode);
> +
>                 wake_up(&fi->page_waitq); /* readpage[s] can wait on fuse wb */
>                 spin_unlock(&fc->lock);
>
> diff --git a/fs/fuse/kio/pcs/fuse_io.c b/fs/fuse/kio/pcs/fuse_io.c
> index 61a2e7f1eac6..00a72c878efb 100644
> --- a/fs/fuse/kio/pcs/fuse_io.c
> +++ b/fs/fuse/kio/pcs/fuse_io.c
> @@ -205,6 +205,7 @@ static void prepare_io_(struct pcs_fuse_req *r, unsigned short type, off_t offse
>  static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>  {
>         struct pcs_fuse_req *r = ioreq->datasource;
> +       struct fuse_conn *fc = cl_from_req(r)->fc;
>
>         BUG_ON(ioreq != &r->exec.io.req);
>
> @@ -217,6 +218,10 @@ static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>                 r->req.out.h.error = 0;
>         }
>
> +       spin_lock(&fc->lock);
> +       list_del_init(&r->req.list);
> +       spin_unlock(&fc->lock);
> +
>         switch (ioreq->type) {
>         case PCS_REQ_T_READ:
>                 on_read_done(r, ioreq->size);
> diff --git a/fs/fuse/kio/pcs/pcs_cluster.c b/fs/fuse/kio/pcs/pcs_cluster.c
> index 5df263f01f98..6b8f38db5c40 100644
> --- a/fs/fuse/kio/pcs/pcs_cluster.c
> +++ b/fs/fuse/kio/pcs/pcs_cluster.c
> @@ -608,6 +608,8 @@ int pcs_cluster_init(struct pcs_fuse_cluster *pfc, struct workqueue_struct *wq,
>         pfc->cc.op.ireq_on_error   = ireq_on_error_;
>         pfc->cc.op.ireq_check_redo = ireq_check_redo_;
>
> +       INIT_LIST_HEAD(&pfc->kio_queue);
> +
>         return 0;
>  }
>
> diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
> index 9c537cb43b30..8a3ae3244b80 100644
> --- a/fs/fuse/kio/pcs/pcs_cluster.h
> +++ b/fs/fuse/kio/pcs/pcs_cluster.h
> @@ -32,6 +32,7 @@ struct pcs_fuse_req {
>  struct pcs_fuse_cluster {
>         struct pcs_cluster_core cc;
>         struct fuse_conn *fc;
> +       struct list_head kio_queue;
>  };
>
>  struct pcs_fuse_work {
> diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
> index cdd4fe578128..1ca27b80b002 100644
> --- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
> +++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
> @@ -983,6 +983,12 @@ error:
>         return;
>
>  submit:
> +       if (!lk)
> +               spin_lock(&pfc->fc->lock);
> +       list_add_tail(&req->list, &pfc->kio_queue);
> +       if (!lk)
> +               spin_unlock(&pfc->fc->lock);
> +
>         if (async)
>                 pcs_cc_submit(ireq->cc, ireq);
>         else
> @@ -1488,6 +1494,15 @@ void __kfuse_trace(struct fuse_conn * fc, unsigned long ip, const char * fmt, ..
>         put_cpu();
>  }
>
> +static void kpcs_kill_requests(struct fuse_conn *fc, struct inode *inode)
> +{
> +       struct pcs_fuse_cluster *pfc = (struct pcs_fuse_cluster*)fc->kio.ctx;
> +
> +       assert_spin_locked(&fc->lock);
> +
> +       return fuse_kill_requests(fc, inode, &pfc->kio_queue);
> +}
> +
>  static struct fuse_kio_ops kio_pcs_ops = {
>         .name           = "pcs",
>         .owner          = THIS_MODULE,
> @@ -1500,6 +1515,7 @@ static struct fuse_kio_ops kio_pcs_ops = {
>         .req_send       = kpcs_req_send,
>         .file_open      = kpcs_file_open,
>         .inode_release  = kpcs_inode_release,
> +       .kill_requests  = kpcs_kill_requests,
>  };
>
>
> --
> 2.15.1
>
Pavel Butsykin Jan. 21, 2019, 4:27 p.m.
I agree fc->lock is evil, but fc->lock is not a necessary condition for
implementation kio queue, so let's look at my second try.

On 18.01.2019 18:00, Alexey Kuznetsov wrote:
> Hello!
> 
> Huh? We were going to get rid of fc->lock and definitely are not going
> to add new ones.
> It is exactly why it was not made in the first place.
> 
> On Fri, Jan 18, 2019 at 4:44 PM Pavel Butsykin <pbutsykin@virtuozzo.com> wrote:
>>
>> Make fuse_invalidate_files() work for kio. This is necessary to maintain
>> vstorage revoke in FPath mode. To do this, let's add a list of inflight kio
>> requests to be able to handle this list in fuse_invalidate_files(). The list
>> will also be useful to implement the full-featured .vstorage.info and
>> fuse_abort_conn() for kio.
>>
>> #VSTOR-19620
>>
>> Signed-off-by: Pavel Butsykin <pbutsykin@virtuozzo.com>
>> ---
>>   fs/fuse/fuse_i.h                   |  4 ++++
>>   fs/fuse/inode.c                    |  8 ++++++--
>>   fs/fuse/kio/pcs/fuse_io.c          |  5 +++++
>>   fs/fuse/kio/pcs/pcs_cluster.c      |  2 ++
>>   fs/fuse/kio/pcs/pcs_cluster.h      |  1 +
>>   fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 16 ++++++++++++++++
>>   6 files changed, 34 insertions(+), 2 deletions(-)
>>
>> diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
>> index 4c27bfa0e74d..0914940d6735 100644
>> --- a/fs/fuse/fuse_i.h
>> +++ b/fs/fuse/fuse_i.h
>> @@ -528,6 +528,7 @@ struct fuse_kio_ops {
>>          int  (*file_open)(struct fuse_conn *fc, struct file *file,
>>                            struct inode *inode);
>>          void (*inode_release)(struct fuse_inode *fi);
>> +       void (*kill_requests)(struct fuse_conn *fc, struct inode *inode);
>>
>>   };
>>   int fuse_register_kio(struct fuse_kio_ops *ops);
>> @@ -1107,4 +1108,7 @@ struct fuse_file *fuse_write_file(struct fuse_conn *fc, struct fuse_inode *fi);
>>
>>   void fuse_release_ff(struct inode *inode, struct fuse_file *ff);
>>
>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>> +                       struct list_head *req_list);
>> +
>>   #endif /* _FS_FUSE_I_H */
>> diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
>> index 34e52262d37e..cb275ff21991 100644
>> --- a/fs/fuse/inode.c
>> +++ b/fs/fuse/inode.c
>> @@ -369,8 +369,8 @@ int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid,
>>          return 0;
>>   }
>>
>> -static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>> -                              struct list_head *req_list)
>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>> +                       struct list_head *req_list)
>>   {
>>          struct fuse_req *req;
>>
>> @@ -393,6 +393,7 @@ static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>                          req->num_pages = 0;
>>                  }
>>   }
>> +EXPORT_SYMBOL_GPL(fuse_kill_requests);
>>
>>   int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>   {
>> @@ -442,6 +443,9 @@ int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>                  }
>>                  fuse_kill_requests(fc, inode, &fc->main_iq.pending);
>>                  fuse_kill_requests(fc, inode, &fc->bg_queue);
>> +               if (fc->kio.op && fc->kio.op->kill_requests)
>> +                       fc->kio.op->kill_requests(fc, inode);
>> +
>>                  wake_up(&fi->page_waitq); /* readpage[s] can wait on fuse wb */
>>                  spin_unlock(&fc->lock);
>>
>> diff --git a/fs/fuse/kio/pcs/fuse_io.c b/fs/fuse/kio/pcs/fuse_io.c
>> index 61a2e7f1eac6..00a72c878efb 100644
>> --- a/fs/fuse/kio/pcs/fuse_io.c
>> +++ b/fs/fuse/kio/pcs/fuse_io.c
>> @@ -205,6 +205,7 @@ static void prepare_io_(struct pcs_fuse_req *r, unsigned short type, off_t offse
>>   static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>   {
>>          struct pcs_fuse_req *r = ioreq->datasource;
>> +       struct fuse_conn *fc = cl_from_req(r)->fc;
>>
>>          BUG_ON(ioreq != &r->exec.io.req);
>>
>> @@ -217,6 +218,10 @@ static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>                  r->req.out.h.error = 0;
>>          }
>>
>> +       spin_lock(&fc->lock);
>> +       list_del_init(&r->req.list);
>> +       spin_unlock(&fc->lock);
>> +
>>          switch (ioreq->type) {
>>          case PCS_REQ_T_READ:
>>                  on_read_done(r, ioreq->size);
>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.c b/fs/fuse/kio/pcs/pcs_cluster.c
>> index 5df263f01f98..6b8f38db5c40 100644
>> --- a/fs/fuse/kio/pcs/pcs_cluster.c
>> +++ b/fs/fuse/kio/pcs/pcs_cluster.c
>> @@ -608,6 +608,8 @@ int pcs_cluster_init(struct pcs_fuse_cluster *pfc, struct workqueue_struct *wq,
>>          pfc->cc.op.ireq_on_error   = ireq_on_error_;
>>          pfc->cc.op.ireq_check_redo = ireq_check_redo_;
>>
>> +       INIT_LIST_HEAD(&pfc->kio_queue);
>> +
>>          return 0;
>>   }
>>
>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
>> index 9c537cb43b30..8a3ae3244b80 100644
>> --- a/fs/fuse/kio/pcs/pcs_cluster.h
>> +++ b/fs/fuse/kio/pcs/pcs_cluster.h
>> @@ -32,6 +32,7 @@ struct pcs_fuse_req {
>>   struct pcs_fuse_cluster {
>>          struct pcs_cluster_core cc;
>>          struct fuse_conn *fc;
>> +       struct list_head kio_queue;
>>   };
>>
>>   struct pcs_fuse_work {
>> diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>> index cdd4fe578128..1ca27b80b002 100644
>> --- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>> +++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>> @@ -983,6 +983,12 @@ error:
>>          return;
>>
>>   submit:
>> +       if (!lk)
>> +               spin_lock(&pfc->fc->lock);
>> +       list_add_tail(&req->list, &pfc->kio_queue);
>> +       if (!lk)
>> +               spin_unlock(&pfc->fc->lock);
>> +
>>          if (async)
>>                  pcs_cc_submit(ireq->cc, ireq);
>>          else
>> @@ -1488,6 +1494,15 @@ void __kfuse_trace(struct fuse_conn * fc, unsigned long ip, const char * fmt, ..
>>          put_cpu();
>>   }
>>
>> +static void kpcs_kill_requests(struct fuse_conn *fc, struct inode *inode)
>> +{
>> +       struct pcs_fuse_cluster *pfc = (struct pcs_fuse_cluster*)fc->kio.ctx;
>> +
>> +       assert_spin_locked(&fc->lock);
>> +
>> +       return fuse_kill_requests(fc, inode, &pfc->kio_queue);
>> +}
>> +
>>   static struct fuse_kio_ops kio_pcs_ops = {
>>          .name           = "pcs",
>>          .owner          = THIS_MODULE,
>> @@ -1500,6 +1515,7 @@ static struct fuse_kio_ops kio_pcs_ops = {
>>          .req_send       = kpcs_req_send,
>>          .file_open      = kpcs_file_open,
>>          .inode_release  = kpcs_inode_release,
>> +       .kill_requests  = kpcs_kill_requests,
>>   };
>>
>>
>> --
>> 2.15.1
>>
Kirill Tkhai Jan. 21, 2019, 4:29 p.m.
JFI: I submitted to mainstream patches for breaking fc->lock dependence:

https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git/log/?h=for-next

I'm going to backport them to us next week.

On 21.01.2019 19:27, Pavel Butsykin wrote:
> I agree fc->lock is evil, but fc->lock is not a necessary condition for
> implementation kio queue, so let's look at my second try.
> 
> On 18.01.2019 18:00, Alexey Kuznetsov wrote:
>> Hello!
>>
>> Huh? We were going to get rid of fc->lock and definitely are not going
>> to add new ones.
>> It is exactly why it was not made in the first place.
>>
>> On Fri, Jan 18, 2019 at 4:44 PM Pavel Butsykin <pbutsykin@virtuozzo.com> wrote:
>>>
>>> Make fuse_invalidate_files() work for kio. This is necessary to maintain
>>> vstorage revoke in FPath mode. To do this, let's add a list of inflight kio
>>> requests to be able to handle this list in fuse_invalidate_files(). The list
>>> will also be useful to implement the full-featured .vstorage.info and
>>> fuse_abort_conn() for kio.
>>>
>>> #VSTOR-19620
>>>
>>> Signed-off-by: Pavel Butsykin <pbutsykin@virtuozzo.com>
>>> ---
>>>   fs/fuse/fuse_i.h                   |  4 ++++
>>>   fs/fuse/inode.c                    |  8 ++++++--
>>>   fs/fuse/kio/pcs/fuse_io.c          |  5 +++++
>>>   fs/fuse/kio/pcs/pcs_cluster.c      |  2 ++
>>>   fs/fuse/kio/pcs/pcs_cluster.h      |  1 +
>>>   fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 16 ++++++++++++++++
>>>   6 files changed, 34 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
>>> index 4c27bfa0e74d..0914940d6735 100644
>>> --- a/fs/fuse/fuse_i.h
>>> +++ b/fs/fuse/fuse_i.h
>>> @@ -528,6 +528,7 @@ struct fuse_kio_ops {
>>>          int  (*file_open)(struct fuse_conn *fc, struct file *file,
>>>                            struct inode *inode);
>>>          void (*inode_release)(struct fuse_inode *fi);
>>> +       void (*kill_requests)(struct fuse_conn *fc, struct inode *inode);
>>>
>>>   };
>>>   int fuse_register_kio(struct fuse_kio_ops *ops);
>>> @@ -1107,4 +1108,7 @@ struct fuse_file *fuse_write_file(struct fuse_conn *fc, struct fuse_inode *fi);
>>>
>>>   void fuse_release_ff(struct inode *inode, struct fuse_file *ff);
>>>
>>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>> +                       struct list_head *req_list);
>>> +
>>>   #endif /* _FS_FUSE_I_H */
>>> diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
>>> index 34e52262d37e..cb275ff21991 100644
>>> --- a/fs/fuse/inode.c
>>> +++ b/fs/fuse/inode.c
>>> @@ -369,8 +369,8 @@ int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid,
>>>          return 0;
>>>   }
>>>
>>> -static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>> -                              struct list_head *req_list)
>>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>> +                       struct list_head *req_list)
>>>   {
>>>          struct fuse_req *req;
>>>
>>> @@ -393,6 +393,7 @@ static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>                          req->num_pages = 0;
>>>                  }
>>>   }
>>> +EXPORT_SYMBOL_GPL(fuse_kill_requests);
>>>
>>>   int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>>   {
>>> @@ -442,6 +443,9 @@ int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>>                  }
>>>                  fuse_kill_requests(fc, inode, &fc->main_iq.pending);
>>>                  fuse_kill_requests(fc, inode, &fc->bg_queue);
>>> +               if (fc->kio.op && fc->kio.op->kill_requests)
>>> +                       fc->kio.op->kill_requests(fc, inode);
>>> +
>>>                  wake_up(&fi->page_waitq); /* readpage[s] can wait on fuse wb */
>>>                  spin_unlock(&fc->lock);
>>>
>>> diff --git a/fs/fuse/kio/pcs/fuse_io.c b/fs/fuse/kio/pcs/fuse_io.c
>>> index 61a2e7f1eac6..00a72c878efb 100644
>>> --- a/fs/fuse/kio/pcs/fuse_io.c
>>> +++ b/fs/fuse/kio/pcs/fuse_io.c
>>> @@ -205,6 +205,7 @@ static void prepare_io_(struct pcs_fuse_req *r, unsigned short type, off_t offse
>>>   static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>>   {
>>>          struct pcs_fuse_req *r = ioreq->datasource;
>>> +       struct fuse_conn *fc = cl_from_req(r)->fc;
>>>
>>>          BUG_ON(ioreq != &r->exec.io.req);
>>>
>>> @@ -217,6 +218,10 @@ static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>>                  r->req.out.h.error = 0;
>>>          }
>>>
>>> +       spin_lock(&fc->lock);
>>> +       list_del_init(&r->req.list);
>>> +       spin_unlock(&fc->lock);
>>> +
>>>          switch (ioreq->type) {
>>>          case PCS_REQ_T_READ:
>>>                  on_read_done(r, ioreq->size);
>>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.c b/fs/fuse/kio/pcs/pcs_cluster.c
>>> index 5df263f01f98..6b8f38db5c40 100644
>>> --- a/fs/fuse/kio/pcs/pcs_cluster.c
>>> +++ b/fs/fuse/kio/pcs/pcs_cluster.c
>>> @@ -608,6 +608,8 @@ int pcs_cluster_init(struct pcs_fuse_cluster *pfc, struct workqueue_struct *wq,
>>>          pfc->cc.op.ireq_on_error   = ireq_on_error_;
>>>          pfc->cc.op.ireq_check_redo = ireq_check_redo_;
>>>
>>> +       INIT_LIST_HEAD(&pfc->kio_queue);
>>> +
>>>          return 0;
>>>   }
>>>
>>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
>>> index 9c537cb43b30..8a3ae3244b80 100644
>>> --- a/fs/fuse/kio/pcs/pcs_cluster.h
>>> +++ b/fs/fuse/kio/pcs/pcs_cluster.h
>>> @@ -32,6 +32,7 @@ struct pcs_fuse_req {
>>>   struct pcs_fuse_cluster {
>>>          struct pcs_cluster_core cc;
>>>          struct fuse_conn *fc;
>>> +       struct list_head kio_queue;
>>>   };
>>>
>>>   struct pcs_fuse_work {
>>> diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>> index cdd4fe578128..1ca27b80b002 100644
>>> --- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>> +++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>> @@ -983,6 +983,12 @@ error:
>>>          return;
>>>
>>>   submit:
>>> +       if (!lk)
>>> +               spin_lock(&pfc->fc->lock);
>>> +       list_add_tail(&req->list, &pfc->kio_queue);
>>> +       if (!lk)
>>> +               spin_unlock(&pfc->fc->lock);
>>> +
>>>          if (async)
>>>                  pcs_cc_submit(ireq->cc, ireq);
>>>          else
>>> @@ -1488,6 +1494,15 @@ void __kfuse_trace(struct fuse_conn * fc, unsigned long ip, const char * fmt, ..
>>>          put_cpu();
>>>   }
>>>
>>> +static void kpcs_kill_requests(struct fuse_conn *fc, struct inode *inode)
>>> +{
>>> +       struct pcs_fuse_cluster *pfc = (struct pcs_fuse_cluster*)fc->kio.ctx;
>>> +
>>> +       assert_spin_locked(&fc->lock);
>>> +
>>> +       return fuse_kill_requests(fc, inode, &pfc->kio_queue);
>>> +}
>>> +
>>>   static struct fuse_kio_ops kio_pcs_ops = {
>>>          .name           = "pcs",
>>>          .owner          = THIS_MODULE,
>>> @@ -1500,6 +1515,7 @@ static struct fuse_kio_ops kio_pcs_ops = {
>>>          .req_send       = kpcs_req_send,
>>>          .file_open      = kpcs_file_open,
>>>          .inode_release  = kpcs_inode_release,
>>> +       .kill_requests  = kpcs_kill_requests,
>>>   };
>>>
>>>
>>> --
>>> 2.15.1
>>>
Pavel Butsykin Jan. 21, 2019, 4:43 p.m.
Good, Thanks!

By the way, is there any response to "Prevent background write requests 
increase inode size" ?

On 21.01.2019 19:29, Kirill Tkhai wrote:
> JFI: I submitted to mainstream patches for breaking fc->lock dependence:
> 
> https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git/log/?h=for-next
> 
> I'm going to backport them to us next week.
> 
> On 21.01.2019 19:27, Pavel Butsykin wrote:
>> I agree fc->lock is evil, but fc->lock is not a necessary condition for
>> implementation kio queue, so let's look at my second try.
>>
>> On 18.01.2019 18:00, Alexey Kuznetsov wrote:
>>> Hello!
>>>
>>> Huh? We were going to get rid of fc->lock and definitely are not going
>>> to add new ones.
>>> It is exactly why it was not made in the first place.
>>>
>>> On Fri, Jan 18, 2019 at 4:44 PM Pavel Butsykin <pbutsykin@virtuozzo.com> wrote:
>>>>
>>>> Make fuse_invalidate_files() work for kio. This is necessary to maintain
>>>> vstorage revoke in FPath mode. To do this, let's add a list of inflight kio
>>>> requests to be able to handle this list in fuse_invalidate_files(). The list
>>>> will also be useful to implement the full-featured .vstorage.info and
>>>> fuse_abort_conn() for kio.
>>>>
>>>> #VSTOR-19620
>>>>
>>>> Signed-off-by: Pavel Butsykin <pbutsykin@virtuozzo.com>
>>>> ---
>>>>    fs/fuse/fuse_i.h                   |  4 ++++
>>>>    fs/fuse/inode.c                    |  8 ++++++--
>>>>    fs/fuse/kio/pcs/fuse_io.c          |  5 +++++
>>>>    fs/fuse/kio/pcs/pcs_cluster.c      |  2 ++
>>>>    fs/fuse/kio/pcs/pcs_cluster.h      |  1 +
>>>>    fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 16 ++++++++++++++++
>>>>    6 files changed, 34 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
>>>> index 4c27bfa0e74d..0914940d6735 100644
>>>> --- a/fs/fuse/fuse_i.h
>>>> +++ b/fs/fuse/fuse_i.h
>>>> @@ -528,6 +528,7 @@ struct fuse_kio_ops {
>>>>           int  (*file_open)(struct fuse_conn *fc, struct file *file,
>>>>                             struct inode *inode);
>>>>           void (*inode_release)(struct fuse_inode *fi);
>>>> +       void (*kill_requests)(struct fuse_conn *fc, struct inode *inode);
>>>>
>>>>    };
>>>>    int fuse_register_kio(struct fuse_kio_ops *ops);
>>>> @@ -1107,4 +1108,7 @@ struct fuse_file *fuse_write_file(struct fuse_conn *fc, struct fuse_inode *fi);
>>>>
>>>>    void fuse_release_ff(struct inode *inode, struct fuse_file *ff);
>>>>
>>>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>> +                       struct list_head *req_list);
>>>> +
>>>>    #endif /* _FS_FUSE_I_H */
>>>> diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
>>>> index 34e52262d37e..cb275ff21991 100644
>>>> --- a/fs/fuse/inode.c
>>>> +++ b/fs/fuse/inode.c
>>>> @@ -369,8 +369,8 @@ int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid,
>>>>           return 0;
>>>>    }
>>>>
>>>> -static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>> -                              struct list_head *req_list)
>>>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>> +                       struct list_head *req_list)
>>>>    {
>>>>           struct fuse_req *req;
>>>>
>>>> @@ -393,6 +393,7 @@ static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>>                           req->num_pages = 0;
>>>>                   }
>>>>    }
>>>> +EXPORT_SYMBOL_GPL(fuse_kill_requests);
>>>>
>>>>    int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>>>    {
>>>> @@ -442,6 +443,9 @@ int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>>>                   }
>>>>                   fuse_kill_requests(fc, inode, &fc->main_iq.pending);
>>>>                   fuse_kill_requests(fc, inode, &fc->bg_queue);
>>>> +               if (fc->kio.op && fc->kio.op->kill_requests)
>>>> +                       fc->kio.op->kill_requests(fc, inode);
>>>> +
>>>>                   wake_up(&fi->page_waitq); /* readpage[s] can wait on fuse wb */
>>>>                   spin_unlock(&fc->lock);
>>>>
>>>> diff --git a/fs/fuse/kio/pcs/fuse_io.c b/fs/fuse/kio/pcs/fuse_io.c
>>>> index 61a2e7f1eac6..00a72c878efb 100644
>>>> --- a/fs/fuse/kio/pcs/fuse_io.c
>>>> +++ b/fs/fuse/kio/pcs/fuse_io.c
>>>> @@ -205,6 +205,7 @@ static void prepare_io_(struct pcs_fuse_req *r, unsigned short type, off_t offse
>>>>    static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>>>    {
>>>>           struct pcs_fuse_req *r = ioreq->datasource;
>>>> +       struct fuse_conn *fc = cl_from_req(r)->fc;
>>>>
>>>>           BUG_ON(ioreq != &r->exec.io.req);
>>>>
>>>> @@ -217,6 +218,10 @@ static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>>>                   r->req.out.h.error = 0;
>>>>           }
>>>>
>>>> +       spin_lock(&fc->lock);
>>>> +       list_del_init(&r->req.list);
>>>> +       spin_unlock(&fc->lock);
>>>> +
>>>>           switch (ioreq->type) {
>>>>           case PCS_REQ_T_READ:
>>>>                   on_read_done(r, ioreq->size);
>>>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.c b/fs/fuse/kio/pcs/pcs_cluster.c
>>>> index 5df263f01f98..6b8f38db5c40 100644
>>>> --- a/fs/fuse/kio/pcs/pcs_cluster.c
>>>> +++ b/fs/fuse/kio/pcs/pcs_cluster.c
>>>> @@ -608,6 +608,8 @@ int pcs_cluster_init(struct pcs_fuse_cluster *pfc, struct workqueue_struct *wq,
>>>>           pfc->cc.op.ireq_on_error   = ireq_on_error_;
>>>>           pfc->cc.op.ireq_check_redo = ireq_check_redo_;
>>>>
>>>> +       INIT_LIST_HEAD(&pfc->kio_queue);
>>>> +
>>>>           return 0;
>>>>    }
>>>>
>>>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
>>>> index 9c537cb43b30..8a3ae3244b80 100644
>>>> --- a/fs/fuse/kio/pcs/pcs_cluster.h
>>>> +++ b/fs/fuse/kio/pcs/pcs_cluster.h
>>>> @@ -32,6 +32,7 @@ struct pcs_fuse_req {
>>>>    struct pcs_fuse_cluster {
>>>>           struct pcs_cluster_core cc;
>>>>           struct fuse_conn *fc;
>>>> +       struct list_head kio_queue;
>>>>    };
>>>>
>>>>    struct pcs_fuse_work {
>>>> diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>>> index cdd4fe578128..1ca27b80b002 100644
>>>> --- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>>> +++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>>> @@ -983,6 +983,12 @@ error:
>>>>           return;
>>>>
>>>>    submit:
>>>> +       if (!lk)
>>>> +               spin_lock(&pfc->fc->lock);
>>>> +       list_add_tail(&req->list, &pfc->kio_queue);
>>>> +       if (!lk)
>>>> +               spin_unlock(&pfc->fc->lock);
>>>> +
>>>>           if (async)
>>>>                   pcs_cc_submit(ireq->cc, ireq);
>>>>           else
>>>> @@ -1488,6 +1494,15 @@ void __kfuse_trace(struct fuse_conn * fc, unsigned long ip, const char * fmt, ..
>>>>           put_cpu();
>>>>    }
>>>>
>>>> +static void kpcs_kill_requests(struct fuse_conn *fc, struct inode *inode)
>>>> +{
>>>> +       struct pcs_fuse_cluster *pfc = (struct pcs_fuse_cluster*)fc->kio.ctx;
>>>> +
>>>> +       assert_spin_locked(&fc->lock);
>>>> +
>>>> +       return fuse_kill_requests(fc, inode, &pfc->kio_queue);
>>>> +}
>>>> +
>>>>    static struct fuse_kio_ops kio_pcs_ops = {
>>>>           .name           = "pcs",
>>>>           .owner          = THIS_MODULE,
>>>> @@ -1500,6 +1515,7 @@ static struct fuse_kio_ops kio_pcs_ops = {
>>>>           .req_send       = kpcs_req_send,
>>>>           .file_open      = kpcs_file_open,
>>>>           .inode_release  = kpcs_inode_release,
>>>> +       .kill_requests  = kpcs_kill_requests,
>>>>    };
>>>>
>>>>
>>>> --
>>>> 2.15.1
>>>>
Kirill Tkhai Jan. 21, 2019, 4:44 p.m.
No

On 21.01.2019 19:43, Pavel Butsykin wrote:
> Good, Thanks!
> 
> By the way, is there any response to "Prevent background write requests 
> increase inode size" ?
> 
> On 21.01.2019 19:29, Kirill Tkhai wrote:
>> JFI: I submitted to mainstream patches for breaking fc->lock dependence:
>>
>> https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git/log/?h=for-next
>>
>> I'm going to backport them to us next week.
>>
>> On 21.01.2019 19:27, Pavel Butsykin wrote:
>>> I agree fc->lock is evil, but fc->lock is not a necessary condition for
>>> implementation kio queue, so let's look at my second try.
>>>
>>> On 18.01.2019 18:00, Alexey Kuznetsov wrote:
>>>> Hello!
>>>>
>>>> Huh? We were going to get rid of fc->lock and definitely are not going
>>>> to add new ones.
>>>> It is exactly why it was not made in the first place.
>>>>
>>>> On Fri, Jan 18, 2019 at 4:44 PM Pavel Butsykin <pbutsykin@virtuozzo.com> wrote:
>>>>>
>>>>> Make fuse_invalidate_files() work for kio. This is necessary to maintain
>>>>> vstorage revoke in FPath mode. To do this, let's add a list of inflight kio
>>>>> requests to be able to handle this list in fuse_invalidate_files(). The list
>>>>> will also be useful to implement the full-featured .vstorage.info and
>>>>> fuse_abort_conn() for kio.
>>>>>
>>>>> #VSTOR-19620
>>>>>
>>>>> Signed-off-by: Pavel Butsykin <pbutsykin@virtuozzo.com>
>>>>> ---
>>>>>    fs/fuse/fuse_i.h                   |  4 ++++
>>>>>    fs/fuse/inode.c                    |  8 ++++++--
>>>>>    fs/fuse/kio/pcs/fuse_io.c          |  5 +++++
>>>>>    fs/fuse/kio/pcs/pcs_cluster.c      |  2 ++
>>>>>    fs/fuse/kio/pcs/pcs_cluster.h      |  1 +
>>>>>    fs/fuse/kio/pcs/pcs_fuse_kdirect.c | 16 ++++++++++++++++
>>>>>    6 files changed, 34 insertions(+), 2 deletions(-)
>>>>>
>>>>> diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
>>>>> index 4c27bfa0e74d..0914940d6735 100644
>>>>> --- a/fs/fuse/fuse_i.h
>>>>> +++ b/fs/fuse/fuse_i.h
>>>>> @@ -528,6 +528,7 @@ struct fuse_kio_ops {
>>>>>           int  (*file_open)(struct fuse_conn *fc, struct file *file,
>>>>>                             struct inode *inode);
>>>>>           void (*inode_release)(struct fuse_inode *fi);
>>>>> +       void (*kill_requests)(struct fuse_conn *fc, struct inode *inode);
>>>>>
>>>>>    };
>>>>>    int fuse_register_kio(struct fuse_kio_ops *ops);
>>>>> @@ -1107,4 +1108,7 @@ struct fuse_file *fuse_write_file(struct fuse_conn *fc, struct fuse_inode *fi);
>>>>>
>>>>>    void fuse_release_ff(struct inode *inode, struct fuse_file *ff);
>>>>>
>>>>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>>> +                       struct list_head *req_list);
>>>>> +
>>>>>    #endif /* _FS_FUSE_I_H */
>>>>> diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
>>>>> index 34e52262d37e..cb275ff21991 100644
>>>>> --- a/fs/fuse/inode.c
>>>>> +++ b/fs/fuse/inode.c
>>>>> @@ -369,8 +369,8 @@ int fuse_reverse_inval_inode(struct super_block *sb, u64 nodeid,
>>>>>           return 0;
>>>>>    }
>>>>>
>>>>> -static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>>> -                              struct list_head *req_list)
>>>>> +void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>>> +                       struct list_head *req_list)
>>>>>    {
>>>>>           struct fuse_req *req;
>>>>>
>>>>> @@ -393,6 +393,7 @@ static void fuse_kill_requests(struct fuse_conn *fc, struct inode *inode,
>>>>>                           req->num_pages = 0;
>>>>>                   }
>>>>>    }
>>>>> +EXPORT_SYMBOL_GPL(fuse_kill_requests);
>>>>>
>>>>>    int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>>>>    {
>>>>> @@ -442,6 +443,9 @@ int fuse_invalidate_files(struct fuse_conn *fc, u64 nodeid)
>>>>>                   }
>>>>>                   fuse_kill_requests(fc, inode, &fc->main_iq.pending);
>>>>>                   fuse_kill_requests(fc, inode, &fc->bg_queue);
>>>>> +               if (fc->kio.op && fc->kio.op->kill_requests)
>>>>> +                       fc->kio.op->kill_requests(fc, inode);
>>>>> +
>>>>>                   wake_up(&fi->page_waitq); /* readpage[s] can wait on fuse wb */
>>>>>                   spin_unlock(&fc->lock);
>>>>>
>>>>> diff --git a/fs/fuse/kio/pcs/fuse_io.c b/fs/fuse/kio/pcs/fuse_io.c
>>>>> index 61a2e7f1eac6..00a72c878efb 100644
>>>>> --- a/fs/fuse/kio/pcs/fuse_io.c
>>>>> +++ b/fs/fuse/kio/pcs/fuse_io.c
>>>>> @@ -205,6 +205,7 @@ static void prepare_io_(struct pcs_fuse_req *r, unsigned short type, off_t offse
>>>>>    static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>>>>    {
>>>>>           struct pcs_fuse_req *r = ioreq->datasource;
>>>>> +       struct fuse_conn *fc = cl_from_req(r)->fc;
>>>>>
>>>>>           BUG_ON(ioreq != &r->exec.io.req);
>>>>>
>>>>> @@ -217,6 +218,10 @@ static void ioreq_complete(pcs_api_iorequest_t *ioreq)
>>>>>                   r->req.out.h.error = 0;
>>>>>           }
>>>>>
>>>>> +       spin_lock(&fc->lock);
>>>>> +       list_del_init(&r->req.list);
>>>>> +       spin_unlock(&fc->lock);
>>>>> +
>>>>>           switch (ioreq->type) {
>>>>>           case PCS_REQ_T_READ:
>>>>>                   on_read_done(r, ioreq->size);
>>>>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.c b/fs/fuse/kio/pcs/pcs_cluster.c
>>>>> index 5df263f01f98..6b8f38db5c40 100644
>>>>> --- a/fs/fuse/kio/pcs/pcs_cluster.c
>>>>> +++ b/fs/fuse/kio/pcs/pcs_cluster.c
>>>>> @@ -608,6 +608,8 @@ int pcs_cluster_init(struct pcs_fuse_cluster *pfc, struct workqueue_struct *wq,
>>>>>           pfc->cc.op.ireq_on_error   = ireq_on_error_;
>>>>>           pfc->cc.op.ireq_check_redo = ireq_check_redo_;
>>>>>
>>>>> +       INIT_LIST_HEAD(&pfc->kio_queue);
>>>>> +
>>>>>           return 0;
>>>>>    }
>>>>>
>>>>> diff --git a/fs/fuse/kio/pcs/pcs_cluster.h b/fs/fuse/kio/pcs/pcs_cluster.h
>>>>> index 9c537cb43b30..8a3ae3244b80 100644
>>>>> --- a/fs/fuse/kio/pcs/pcs_cluster.h
>>>>> +++ b/fs/fuse/kio/pcs/pcs_cluster.h
>>>>> @@ -32,6 +32,7 @@ struct pcs_fuse_req {
>>>>>    struct pcs_fuse_cluster {
>>>>>           struct pcs_cluster_core cc;
>>>>>           struct fuse_conn *fc;
>>>>> +       struct list_head kio_queue;
>>>>>    };
>>>>>
>>>>>    struct pcs_fuse_work {
>>>>> diff --git a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>>>> index cdd4fe578128..1ca27b80b002 100644
>>>>> --- a/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>>>> +++ b/fs/fuse/kio/pcs/pcs_fuse_kdirect.c
>>>>> @@ -983,6 +983,12 @@ error:
>>>>>           return;
>>>>>
>>>>>    submit:
>>>>> +       if (!lk)
>>>>> +               spin_lock(&pfc->fc->lock);
>>>>> +       list_add_tail(&req->list, &pfc->kio_queue);
>>>>> +       if (!lk)
>>>>> +               spin_unlock(&pfc->fc->lock);
>>>>> +
>>>>>           if (async)
>>>>>                   pcs_cc_submit(ireq->cc, ireq);
>>>>>           else
>>>>> @@ -1488,6 +1494,15 @@ void __kfuse_trace(struct fuse_conn * fc, unsigned long ip, const char * fmt, ..
>>>>>           put_cpu();
>>>>>    }
>>>>>
>>>>> +static void kpcs_kill_requests(struct fuse_conn *fc, struct inode *inode)
>>>>> +{
>>>>> +       struct pcs_fuse_cluster *pfc = (struct pcs_fuse_cluster*)fc->kio.ctx;
>>>>> +
>>>>> +       assert_spin_locked(&fc->lock);
>>>>> +
>>>>> +       return fuse_kill_requests(fc, inode, &pfc->kio_queue);
>>>>> +}
>>>>> +
>>>>>    static struct fuse_kio_ops kio_pcs_ops = {
>>>>>           .name           = "pcs",
>>>>>           .owner          = THIS_MODULE,
>>>>> @@ -1500,6 +1515,7 @@ static struct fuse_kio_ops kio_pcs_ops = {
>>>>>           .req_send       = kpcs_req_send,
>>>>>           .file_open      = kpcs_file_open,
>>>>>           .inode_release  = kpcs_inode_release,
>>>>> +       .kill_requests  = kpcs_kill_requests,
>>>>>    };
>>>>>
>>>>>
>>>>> --
>>>>> 2.15.1
>>>>>