[v10,6/9] ve/cgroup: unmark ve-root cgroups at container stop

Submitted by Valeriy Vdovin on April 16, 2020, 4:14 p.m.

Details

Message ID 1587053651-246365-7-git-send-email-valeriy.vdovin@virtuozzo.com
State New
Series "Make release_agent per-cgroup property. Run release_agent in proper ve."
Headers show

Commit Message

Valeriy Vdovin April 16, 2020, 4:14 p.m.
Signed-off-by: Valeriy Vdovin <valeriy.vdovin@virtuozzo.com>
Reviewed-by: Kirill Tkhai <ktkhai@virtuozzo.com>
---
 include/linux/cgroup.h |  1 +
 kernel/cgroup.c        | 42 ++++++++++++++++++++++++++++++++++++++++++
 kernel/ve/ve.c         |  2 ++
 3 files changed, 45 insertions(+)

Patch hide | download patch | download mbox

diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 0a42f93..1bd0fe7 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -636,6 +636,7 @@  int cgroup_task_count(const struct cgroup *cgrp);
 
 #ifdef CONFIG_VE
 void cgroup_mark_ve_roots(struct ve_struct *ve);
+void cgroup_unmark_ve_roots(struct ve_struct *ve);
 #endif
 
 /*
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 60a4ca4..1dd36ed 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -635,6 +635,35 @@  static struct css_set *find_css_set(
 }
 
 /*
+ * Walk each cgroup link of a given css_set and find a cgroup that
+ * is the child of cgroupfs_root in argument.
+ */
+static struct cgroup *css_cgroup_from_root(struct css_set *css_set,
+					    struct cgroupfs_root *root)
+{
+	struct cgroup *res = NULL;
+
+	BUG_ON(!mutex_is_locked(&cgroup_mutex));
+	read_lock(&css_set_lock);
+
+	if (css_set == &init_css_set) {
+		res = &root->top_cgroup;
+	} else {
+		struct cg_cgroup_link *link;
+		list_for_each_entry(link, &css_set->cg_links, cg_link_list) {
+			struct cgroup *c = link->cgrp;
+			if (c->root == root) {
+				res = c;
+				break;
+			}
+		}
+	}
+	read_unlock(&css_set_lock);
+	BUG_ON(!res);
+	return res;
+}
+
+/*
  * Return the cgroup for "task" from the given hierarchy. Must be
  * called with cgroup_mutex held.
  */
@@ -4237,6 +4266,19 @@  void cgroup_mark_ve_roots(struct ve_struct *ve)
 	mutex_unlock(&cgroup_mutex);
 }
 
+void cgroup_unmark_ve_roots(struct ve_struct *ve)
+{
+	struct cgroup *cgrp;
+	struct cgroupfs_root *root;
+
+	mutex_lock(&cgroup_mutex);
+	for_each_active_root(root) {
+		cgrp = css_cgroup_from_root(ve->root_css_set, root);
+		clear_bit(CGRP_VE_ROOT, &cgrp->flags);
+	}
+	mutex_unlock(&cgroup_mutex);
+}
+
 struct cgroup *cgroup_get_ve_root(struct cgroup *cgrp)
 {
 	struct cgroup *ve_root = NULL;
diff --git a/kernel/ve/ve.c b/kernel/ve/ve.c
index eedb85d..db3b600 100644
--- a/kernel/ve/ve.c
+++ b/kernel/ve/ve.c
@@ -618,6 +618,8 @@  void ve_exit_ns(struct pid_namespace *pid_ns)
 	if (!ve->ve_ns || ve->ve_ns->pid_ns != pid_ns)
 		return;
 
+	cgroup_unmark_ve_roots(ve);
+
 	ve_workqueue_stop(ve);
 
 	/*

Comments

Kirill Tkhai April 17, 2020, 7:42 a.m.
On 16.04.2020 19:14, Valeriy Vdovin wrote:
> Signed-off-by: Valeriy Vdovin <valeriy.vdovin@virtuozzo.com>
> Reviewed-by: Kirill Tkhai <ktkhai@virtuozzo.com>

1)You have to add my Reviewed-by tags only for patches, where I've really given it.
There were not my ack to patches 6-9/9.

2)Have you read my answer to "Re: [PATCH v8 6/9] ve/cgroup: unmark ve-root cgroups at container stop"?

> ---
>  include/linux/cgroup.h |  1 +
>  kernel/cgroup.c        | 42 ++++++++++++++++++++++++++++++++++++++++++
>  kernel/ve/ve.c         |  2 ++
>  3 files changed, 45 insertions(+)
> 
> diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
> index 0a42f93..1bd0fe7 100644
> --- a/include/linux/cgroup.h
> +++ b/include/linux/cgroup.h
> @@ -636,6 +636,7 @@ int cgroup_task_count(const struct cgroup *cgrp);
>  
>  #ifdef CONFIG_VE
>  void cgroup_mark_ve_roots(struct ve_struct *ve);
> +void cgroup_unmark_ve_roots(struct ve_struct *ve);
>  #endif
>  
>  /*
> diff --git a/kernel/cgroup.c b/kernel/cgroup.c
> index 60a4ca4..1dd36ed 100644
> --- a/kernel/cgroup.c
> +++ b/kernel/cgroup.c
> @@ -635,6 +635,35 @@ static struct css_set *find_css_set(
>  }
>  
>  /*
> + * Walk each cgroup link of a given css_set and find a cgroup that
> + * is the child of cgroupfs_root in argument.
> + */
> +static struct cgroup *css_cgroup_from_root(struct css_set *css_set,
> +					    struct cgroupfs_root *root)
> +{
> +	struct cgroup *res = NULL;
> +
> +	BUG_ON(!mutex_is_locked(&cgroup_mutex));
> +	read_lock(&css_set_lock);
> +
> +	if (css_set == &init_css_set) {
> +		res = &root->top_cgroup;
> +	} else {
> +		struct cg_cgroup_link *link;
> +		list_for_each_entry(link, &css_set->cg_links, cg_link_list) {
> +			struct cgroup *c = link->cgrp;
> +			if (c->root == root) {
> +				res = c;
> +				break;
> +			}
> +		}
> +	}
> +	read_unlock(&css_set_lock);
> +	BUG_ON(!res);
> +	return res;
> +}
> +
> +/*
>   * Return the cgroup for "task" from the given hierarchy. Must be
>   * called with cgroup_mutex held.
>   */
> @@ -4237,6 +4266,19 @@ void cgroup_mark_ve_roots(struct ve_struct *ve)
>  	mutex_unlock(&cgroup_mutex);
>  }
>  
> +void cgroup_unmark_ve_roots(struct ve_struct *ve)
> +{
> +	struct cgroup *cgrp;
> +	struct cgroupfs_root *root;
> +
> +	mutex_lock(&cgroup_mutex);
> +	for_each_active_root(root) {
> +		cgrp = css_cgroup_from_root(ve->root_css_set, root);
> +		clear_bit(CGRP_VE_ROOT, &cgrp->flags);
> +	}
> +	mutex_unlock(&cgroup_mutex);
> +}
> +
>  struct cgroup *cgroup_get_ve_root(struct cgroup *cgrp)
>  {
>  	struct cgroup *ve_root = NULL;
> diff --git a/kernel/ve/ve.c b/kernel/ve/ve.c
> index eedb85d..db3b600 100644
> --- a/kernel/ve/ve.c
> +++ b/kernel/ve/ve.c
> @@ -618,6 +618,8 @@ void ve_exit_ns(struct pid_namespace *pid_ns)
>  	if (!ve->ve_ns || ve->ve_ns->pid_ns != pid_ns)
>  		return;
>  
> +	cgroup_unmark_ve_roots(ve);
> +
>  	ve_workqueue_stop(ve);
>  
>  	/*
>
Pavel Tikhomirov April 17, 2020, 7:59 a.m.
> @@ -4237,6 +4266,19 @@ void cgroup_mark_ve_roots(struct ve_struct *ve)
>   	mutex_unlock(&cgroup_mutex);
>   }
>   
> +void cgroup_unmark_ve_roots(struct ve_struct *ve)
> +{
> +	struct cgroup *cgrp;
> +	struct cgroupfs_root *root;
> +
> +	mutex_lock(&cgroup_mutex);
> +	for_each_active_root(root) {
> +		cgrp = css_cgroup_from_root(ve->root_css_set, root);

Maybe we can use this new helper in cgroup_mark_ve_roots too for 
consistency (maybe in separate patch), what do you think?

> +		clear_bit(CGRP_VE_ROOT, &cgrp->flags);
> +	}
> +	mutex_unlock(&cgroup_mutex);
> +}
> +
>   struct cgroup *cgroup_get_ve_root(struct cgroup *cgrp)
>   {
>   	struct cgroup *ve_root = NULL;
Valeriy Vdovin April 17, 2020, 8:18 a.m.
>1)You have to add my Reviewed-by tags only for patches, where I've really given it.

My bad, I've done a poor job of matching them in reply series from you. I've definitely misinterpreted your series of replies yesterday.

>There were not my ack to patches 6-9/9.
Ok, I see it now.

>2)Have you read my answer to "Re: [PATCH v8 6/9] ve/cgroup: unmark ve-root cgroups at container stop"?
>>How can we access ve_owner when it is not defined yet? It goes in next patch.
>>Can we move ve->ve_owner declaration and assignment in this patch?

Sure. I've fixed that.
Kirill Tkhai April 17, 2020, 9:15 a.m.
On 16.04.2020 19:14, Valeriy Vdovin wrote:
> Signed-off-by: Valeriy Vdovin <valeriy.vdovin@virtuozzo.com>
> Reviewed-by: Kirill Tkhai <ktkhai@virtuozzo.com>
> ---
>  include/linux/cgroup.h |  1 +
>  kernel/cgroup.c        | 42 ++++++++++++++++++++++++++++++++++++++++++
>  kernel/ve/ve.c         |  2 ++
>  3 files changed, 45 insertions(+)
> 
> diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
> index 0a42f93..1bd0fe7 100644
> --- a/include/linux/cgroup.h
> +++ b/include/linux/cgroup.h
> @@ -636,6 +636,7 @@ int cgroup_task_count(const struct cgroup *cgrp);
>  
>  #ifdef CONFIG_VE
>  void cgroup_mark_ve_roots(struct ve_struct *ve);
> +void cgroup_unmark_ve_roots(struct ve_struct *ve);
>  #endif
>  
>  /*
> diff --git a/kernel/cgroup.c b/kernel/cgroup.c
> index 60a4ca4..1dd36ed 100644
> --- a/kernel/cgroup.c
> +++ b/kernel/cgroup.c
> @@ -635,6 +635,35 @@ static struct css_set *find_css_set(
>  }
>  
>  /*
> + * Walk each cgroup link of a given css_set and find a cgroup that
> + * is the child of cgroupfs_root in argument.
> + */
> +static struct cgroup *css_cgroup_from_root(struct css_set *css_set,
> +					    struct cgroupfs_root *root)
> +{
> +	struct cgroup *res = NULL;
> +
> +	BUG_ON(!mutex_is_locked(&cgroup_mutex));
> +	read_lock(&css_set_lock);
> +
> +	if (css_set == &init_css_set) {
> +		res = &root->top_cgroup;

Will the function work correctly in case jf we completely remove above "if"?
Is this "if" optimization for init_css_set?

> +	} else {
> +		struct cg_cgroup_link *link;
> +		list_for_each_entry(link, &css_set->cg_links, cg_link_list) {
> +			struct cgroup *c = link->cgrp;
> +			if (c->root == root) {
> +				res = c;
> +				break;
> +			}
> +		}
> +	}
> +	read_unlock(&css_set_lock);
> +	BUG_ON(!res);
> +	return res;
> +}
> +
> +/*
>   * Return the cgroup for "task" from the given hierarchy. Must be
>   * called with cgroup_mutex held.
>   */
> @@ -4237,6 +4266,19 @@ void cgroup_mark_ve_roots(struct ve_struct *ve)
>  	mutex_unlock(&cgroup_mutex);
>  }
>  
> +void cgroup_unmark_ve_roots(struct ve_struct *ve)
> +{
> +	struct cgroup *cgrp;
> +	struct cgroupfs_root *root;
> +
> +	mutex_lock(&cgroup_mutex);
> +	for_each_active_root(root) {
> +		cgrp = css_cgroup_from_root(ve->root_css_set, root);
> +		clear_bit(CGRP_VE_ROOT, &cgrp->flags);
> +	}
> +	mutex_unlock(&cgroup_mutex);
> +}
> +
>  struct cgroup *cgroup_get_ve_root(struct cgroup *cgrp)
>  {
>  	struct cgroup *ve_root = NULL;
> diff --git a/kernel/ve/ve.c b/kernel/ve/ve.c
> index eedb85d..db3b600 100644
> --- a/kernel/ve/ve.c
> +++ b/kernel/ve/ve.c
> @@ -618,6 +618,8 @@ void ve_exit_ns(struct pid_namespace *pid_ns)
>  	if (!ve->ve_ns || ve->ve_ns->pid_ns != pid_ns)
>  		return;
>  
> +	cgroup_unmark_ve_roots(ve);
> +
>  	ve_workqueue_stop(ve);
>  
>  	/*
>
Valeriy Vdovin April 17, 2020, 12:11 p.m.
>Will the function work correctly in case jf we completely remove above "if"? Is this "if" optimization for init_css_set?

This is indeed an optimization. And also it looks like it's the most common case.
Kirill Tkhai April 17, 2020, 12:14 p.m.
On 17.04.2020 15:11, Valeriy Vdovin wrote:
>>Will the function work correctly in case jf we completely remove above "if"? Is this "if" optimization for init_css_set?
> 
> This is indeed an optimization. And also it looks like it's the most common case.

(Please, avoid top-posting in your replies. Top-posting is not welcomed. See in Documentation/development-process/2.Process)

Then, we should add a comment about that this is optimization, I think.

------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
> *From:* Kirill Tkhai <ktkhai@virtuozzo.com>
> *Sent:* Friday, April 17, 2020 12:15 PM
> *To:* Valeriy Vdovin <Valeriy.Vdovin@virtuozzo.com>; devel@openvz.org <devel@openvz.org>; Konstantin Khorenko <khorenko@virtuozzo.com>; Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
> *Subject:* Re: [PATCH v10 6/9] ve/cgroup: unmark ve-root cgroups at container stop
>  
> On 16.04.2020 19:14, Valeriy Vdovin wrote:
>> Signed-off-by: Valeriy Vdovin <valeriy.vdovin@virtuozzo.com>
>> Reviewed-by: Kirill Tkhai <ktkhai@virtuozzo.com>
>> ---
>>  include/linux/cgroup.h |  1 +
>>  kernel/cgroup.c        | 42 ++++++++++++++++++++++++++++++++++++++++++
>>  kernel/ve/ve.c         |  2 ++
>>  3 files changed, 45 insertions(+)
>> 
>> diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
>> index 0a42f93..1bd0fe7 100644
>> --- a/include/linux/cgroup.h
>> +++ b/include/linux/cgroup.h
>> @@ -636,6 +636,7 @@ int cgroup_task_count(const struct cgroup *cgrp);
>>  
>>  #ifdef CONFIG_VE
>>  void cgroup_mark_ve_roots(struct ve_struct *ve);
>> +void cgroup_unmark_ve_roots(struct ve_struct *ve);
>>  #endif
>>  
>>  /*
>> diff --git a/kernel/cgroup.c b/kernel/cgroup.c
>> index 60a4ca4..1dd36ed 100644
>> --- a/kernel/cgroup.c
>> +++ b/kernel/cgroup.c
>> @@ -635,6 +635,35 @@ static struct css_set *find_css_set(
>>  }
>>  
>>  /*
>> + * Walk each cgroup link of a given css_set and find a cgroup that
>> + * is the child of cgroupfs_root in argument.
>> + */
>> +static struct cgroup *css_cgroup_from_root(struct css_set *css_set,
>> +                                         struct cgroupfs_root *root)
>> +{
>> +     struct cgroup *res = NULL;
>> +
>> +     BUG_ON(!mutex_is_locked(&cgroup_mutex));
>> +     read_lock(&css_set_lock);
>> +
>> +     if (css_set == &init_css_set) {
>> +             res = &root->top_cgroup;
> 
> Will the function work correctly in case jf we completely remove above "if"?
> Is this "if" optimization for init_css_set?
> 
>> +     } else {
>> +             struct cg_cgroup_link *link;
>> +             list_for_each_entry(link, &css_set->cg_links, cg_link_list) {
>> +                     struct cgroup *c = link->cgrp;
>> +                     if (c->root == root) {
>> +                             res = c;
>> +                             break;
>> +                     }
>> +             }
>> +     }
>> +     read_unlock(&css_set_lock);
>> +     BUG_ON(!res);
>> +     return res;
>> +}
>> +
>> +/*
>>   * Return the cgroup for "task" from the given hierarchy. Must be
>>   * called with cgroup_mutex held.
>>   */
>> @@ -4237,6 +4266,19 @@ void cgroup_mark_ve_roots(struct ve_struct *ve)
>>        mutex_unlock(&cgroup_mutex);
>>  }
>>  
>> +void cgroup_unmark_ve_roots(struct ve_struct *ve)
>> +{
>> +     struct cgroup *cgrp;
>> +     struct cgroupfs_root *root;
>> +
>> +     mutex_lock(&cgroup_mutex);
>> +     for_each_active_root(root) {
>> +             cgrp = css_cgroup_from_root(ve->root_css_set, root);
>> +             clear_bit(CGRP_VE_ROOT, &cgrp->flags);
>> +     }
>> +     mutex_unlock(&cgroup_mutex);
>> +}
>> +
>>  struct cgroup *cgroup_get_ve_root(struct cgroup *cgrp)
>>  {
>>        struct cgroup *ve_root = NULL;
>> diff --git a/kernel/ve/ve.c b/kernel/ve/ve.c
>> index eedb85d..db3b600 100644
>> --- a/kernel/ve/ve.c
>> +++ b/kernel/ve/ve.c
>> @@ -618,6 +618,8 @@ void ve_exit_ns(struct pid_namespace *pid_ns)
>>        if (!ve->ve_ns || ve->ve_ns->pid_ns != pid_ns)
>>                return;
>>  
>> +     cgroup_unmark_ve_roots(ve);
>> +
>>        ve_workqueue_stop(ve);
>>  
>>        /*
>> 
>
Kirill Tkhai April 17, 2020, 12:17 p.m.
On 17.04.2020 15:14, Kirill Tkhai wrote:
> On 17.04.2020 15:11, Valeriy Vdovin wrote:
>>> Will the function work correctly in case jf we completely remove above "if"? Is this "if" optimization for init_css_set?
>>
>> This is indeed an optimization. And also it looks like it's the most common case.
> 
> (Please, avoid top-posting in your replies. Top-posting is not welcomed. See in Documentation/development-process/2.Process)
> 
> Then, we should add a comment about that this is optimization, I think.

With an appropriate comment about this is optimization, you may add my Reviewed-by: to this patch in next resend.

------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>> *From:* Kirill Tkhai <ktkhai@virtuozzo.com>
>> *Sent:* Friday, April 17, 2020 12:15 PM
>> *To:* Valeriy Vdovin <Valeriy.Vdovin@virtuozzo.com>; devel@openvz.org <devel@openvz.org>; Konstantin Khorenko <khorenko@virtuozzo.com>; Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
>> *Subject:* Re: [PATCH v10 6/9] ve/cgroup: unmark ve-root cgroups at container stop
>>  
>> On 16.04.2020 19:14, Valeriy Vdovin wrote:
>>> Signed-off-by: Valeriy Vdovin <valeriy.vdovin@virtuozzo.com>
>>> Reviewed-by: Kirill Tkhai <ktkhai@virtuozzo.com>
>>> ---
>>>   include/linux/cgroup.h |  1 +
>>>   kernel/cgroup.c        | 42 ++++++++++++++++++++++++++++++++++++++++++
>>>   kernel/ve/ve.c         |  2 ++
>>>   3 files changed, 45 insertions(+)
>>>
>>> diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
>>> index 0a42f93..1bd0fe7 100644
>>> --- a/include/linux/cgroup.h
>>> +++ b/include/linux/cgroup.h
>>> @@ -636,6 +636,7 @@ int cgroup_task_count(const struct cgroup *cgrp);
>>>   
>>>   #ifdef CONFIG_VE
>>>   void cgroup_mark_ve_roots(struct ve_struct *ve);
>>> +void cgroup_unmark_ve_roots(struct ve_struct *ve);
>>>   #endif
>>>   
>>>   /*
>>> diff --git a/kernel/cgroup.c b/kernel/cgroup.c
>>> index 60a4ca4..1dd36ed 100644
>>> --- a/kernel/cgroup.c
>>> +++ b/kernel/cgroup.c
>>> @@ -635,6 +635,35 @@ static struct css_set *find_css_set(
>>>   }
>>>   
>>>   /*
>>> + * Walk each cgroup link of a given css_set and find a cgroup that
>>> + * is the child of cgroupfs_root in argument.
>>> + */
>>> +static struct cgroup *css_cgroup_from_root(struct css_set *css_set,
>>> +                                         struct cgroupfs_root *root)
>>> +{
>>> +     struct cgroup *res = NULL;
>>> +
>>> +     BUG_ON(!mutex_is_locked(&cgroup_mutex));
>>> +     read_lock(&css_set_lock);
>>> +
>>> +     if (css_set == &init_css_set) {
>>> +             res = &root->top_cgroup;
>>
>> Will the function work correctly in case jf we completely remove above "if"?
>> Is this "if" optimization for init_css_set?
>>
>>> +     } else {
>>> +             struct cg_cgroup_link *link;
>>> +             list_for_each_entry(link, &css_set->cg_links, cg_link_list) {
>>> +                     struct cgroup *c = link->cgrp;
>>> +                     if (c->root == root) {
>>> +                             res = c;
>>> +                             break;
>>> +                     }
>>> +             }
>>> +     }
>>> +     read_unlock(&css_set_lock);
>>> +     BUG_ON(!res);
>>> +     return res;
>>> +}
>>> +
>>> +/*
>>>    * Return the cgroup for "task" from the given hierarchy. Must be
>>>    * called with cgroup_mutex held.
>>>    */
>>> @@ -4237,6 +4266,19 @@ void cgroup_mark_ve_roots(struct ve_struct *ve)
>>>         mutex_unlock(&cgroup_mutex);
>>>   }
>>>   
>>> +void cgroup_unmark_ve_roots(struct ve_struct *ve)
>>> +{
>>> +     struct cgroup *cgrp;
>>> +     struct cgroupfs_root *root;
>>> +
>>> +     mutex_lock(&cgroup_mutex);
>>> +     for_each_active_root(root) {
>>> +             cgrp = css_cgroup_from_root(ve->root_css_set, root);
>>> +             clear_bit(CGRP_VE_ROOT, &cgrp->flags);
>>> +     }
>>> +     mutex_unlock(&cgroup_mutex);
>>> +}
>>> +
>>>   struct cgroup *cgroup_get_ve_root(struct cgroup *cgrp)
>>>   {
>>>         struct cgroup *ve_root = NULL;
>>> diff --git a/kernel/ve/ve.c b/kernel/ve/ve.c
>>> index eedb85d..db3b600 100644
>>> --- a/kernel/ve/ve.c
>>> +++ b/kernel/ve/ve.c
>>> @@ -618,6 +618,8 @@ void ve_exit_ns(struct pid_namespace *pid_ns)
>>>         if (!ve->ve_ns || ve->ve_ns->pid_ns != pid_ns)
>>>                 return;
>>>   
>>> +     cgroup_unmark_ve_roots(ve);
>>> +
>>>         ve_workqueue_stop(ve);
>>>   
>>>         /*
>>>
>>
>