early_init: Raise nr_open limit on particular commands

Submitted by Cyrill Gorcunov on March 27, 2019, 5:24 p.m.

Details

Message ID 20190327172430.15911-1-gorcunov@gmail.com
State New
Series "early_init: Raise nr_open limit on particular commands"
Headers show

Commit Message

Cyrill Gorcunov March 27, 2019, 5:24 p.m.
For swrk, service, dump, restore we need to to raise
nr_file limit to be able to process containers with
huge number of files opened.

Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
---
 criu/cr-restore.c | 32 ------------------------
 criu/crtools.c    | 63 +++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 34 deletions(-)

Patch hide | download patch | download mbox

diff --git a/criu/cr-restore.c b/criu/cr-restore.c
index af2ca2921d00..a6cb8f1b3e07 100644
--- a/criu/cr-restore.c
+++ b/criu/cr-restore.c
@@ -2327,42 +2327,10 @@  int prepare_dummy_task_state(struct pstree_item *pi)
 	return 0;
 }
 
-static void rlimit_unlimit_nofile_self(void)
-{
-	struct rlimit new;
-
-	new.rlim_cur = kdat.sysctl_nr_open;
-	new.rlim_max = kdat.sysctl_nr_open;
-
-	if (prlimit(getpid(), RLIMIT_NOFILE, &new, NULL)) {
-		pr_perror("rlimir: Can't setup RLIMIT_NOFILE for self");
-		return;
-	} else
-		pr_debug("rlimit: RLIMIT_NOFILE unlimited for self\n");
-	service_fd_rlim_cur = kdat.sysctl_nr_open;
-}
-
 int cr_restore_tasks(void)
 {
 	int ret = -1;
 
-	/*
-	 * Service fd engine implies that file descriptors
-	 * used won't be borrowed by the rest of the code
-	 * and default 1024 limit is not enough for high
-	 * loaded test/containers. Thus use kdat engine
-	 * to fetch current system level limit for numbers
-	 * of files allowed to open up and lift up own
-	 * limits.
-	 *
-	 * Note we have to do it before the service fd
-	 * get inited and we dont exit with errors here
-	 * because in worst scenario where clash of fd
-	 * happen we simply exit with explicit error
-	 * during real action stage.
-	 */
-	rlimit_unlimit_nofile_self();
-
 	if (cr_plugin_init(CR_PLUGIN_STAGE__RESTORE))
 		return -1;
 
diff --git a/criu/crtools.c b/criu/crtools.c
index 3f64000c1e87..3654aa3229de 100644
--- a/criu/crtools.c
+++ b/criu/crtools.c
@@ -51,8 +51,67 @@ 
 #include "sysctl.h"
 #include "img-remote.h"
 
-static int early_init(void)
+static void rlimit_unlimit_nofile(void)
 {
+	struct rlimit new;
+
+	new.rlim_cur = kdat.sysctl_nr_open;
+	new.rlim_max = kdat.sysctl_nr_open;
+
+	if (prlimit(getpid(), RLIMIT_NOFILE, &new, NULL)) {
+		pr_perror("rlimit: Can't setup RLIMIT_NOFILE for self");
+		return;
+	} else
+		pr_debug("rlimit: RLIMIT_NOFILE unlimited for self\n");
+
+	service_fd_rlim_cur = kdat.sysctl_nr_open;
+}
+
+static int early_init(const char *cmd)
+{
+	static const char *nofile_cmds[] = {
+		"swrk", "service",
+		"dump", "restore",
+	};
+	size_t i;
+
+	/*
+	 * Service fd engine implies that file descriptors
+	 * used won't be borrowed by the rest of the code
+	 * and default 1024 limit is not enough for high
+	 * loaded test/containers. Thus use kdat engine
+	 * to fetch current system level limit for numbers
+	 * of files allowed to open up and lift up own
+	 * limits.
+	 *
+	 * Note we have to do it before the service fd
+	 * get inited and we dont exit with errors here
+	 * because in worst scenario where clash of fd
+	 * happen we simply exit with explicit error
+	 * during real action stage.
+	 *
+	 * Same time raising limits cause kernel fdtable
+	 * to bloat so we do this only on the @nofile_cmds:
+	 *
+	 *  - on dump criu needs additional files for sfd,
+	 *    thus if container already has many files opened
+	 *    we need to have at least not less space when
+	 *    fetching fds from a target process,
+	 *    on restore the same -- we need to be able to
+	 *    allocate all files from images;
+	 *
+	 *  - swrk and servise obtain requests on the fly,
+	 *    thus we don't know if dump/restore will be
+	 *    there.
+	 */
+	for (i = 0; i < ARRAY_SIZE(nofile_cmds); i++) {
+		if (strcmp(nofile_cmds[i], cmd))
+			continue;
+		if (!kerndat_files_stat(true))
+			rlimit_unlimit_nofile();
+		break;
+	}
+
 	if (init_service_fd())
 		return 1;
 
@@ -94,7 +153,7 @@  int main(int argc, char *argv[], char *envp[])
 
 	log_set_loglevel(opts.log_level);
 
-	if (early_init())
+	if (early_init(argv[optind]))
 		return -1;
 
 	if (!strcmp(argv[1], "swrk")) {

Comments

Radostin Stoyanov April 5, 2019, 8:40 p.m.
ping

On 27/03/2019 17:24, Cyrill Gorcunov wrote:
> For swrk, service, dump, restore we need to to raise
> nr_file limit to be able to process containers with
> huge number of files opened.
>
> Signed-off-by: Cyrill Gorcunov <gorcunov@gmail.com>
> ---
>  criu/cr-restore.c | 32 ------------------------
>  criu/crtools.c    | 63 +++++++++++++++++++++++++++++++++++++++++++++--
>  2 files changed, 61 insertions(+), 34 deletions(-)
>
> diff --git a/criu/cr-restore.c b/criu/cr-restore.c
> index af2ca2921d00..a6cb8f1b3e07 100644
> --- a/criu/cr-restore.c
> +++ b/criu/cr-restore.c
> @@ -2327,42 +2327,10 @@ int prepare_dummy_task_state(struct pstree_item *pi)
>  	return 0;
>  }
>  
> -static void rlimit_unlimit_nofile_self(void)
> -{
> -	struct rlimit new;
> -
> -	new.rlim_cur = kdat.sysctl_nr_open;
> -	new.rlim_max = kdat.sysctl_nr_open;
> -
> -	if (prlimit(getpid(), RLIMIT_NOFILE, &new, NULL)) {
> -		pr_perror("rlimir: Can't setup RLIMIT_NOFILE for self");
> -		return;
> -	} else
> -		pr_debug("rlimit: RLIMIT_NOFILE unlimited for self\n");
> -	service_fd_rlim_cur = kdat.sysctl_nr_open;
> -}
> -
>  int cr_restore_tasks(void)
>  {
>  	int ret = -1;
>  
> -	/*
> -	 * Service fd engine implies that file descriptors
> -	 * used won't be borrowed by the rest of the code
> -	 * and default 1024 limit is not enough for high
> -	 * loaded test/containers. Thus use kdat engine
> -	 * to fetch current system level limit for numbers
> -	 * of files allowed to open up and lift up own
> -	 * limits.
> -	 *
> -	 * Note we have to do it before the service fd
> -	 * get inited and we dont exit with errors here
> -	 * because in worst scenario where clash of fd
> -	 * happen we simply exit with explicit error
> -	 * during real action stage.
> -	 */
> -	rlimit_unlimit_nofile_self();
> -
>  	if (cr_plugin_init(CR_PLUGIN_STAGE__RESTORE))
>  		return -1;
>  
> diff --git a/criu/crtools.c b/criu/crtools.c
> index 3f64000c1e87..3654aa3229de 100644
> --- a/criu/crtools.c
> +++ b/criu/crtools.c
> @@ -51,8 +51,67 @@
>  #include "sysctl.h"
>  #include "img-remote.h"
>  
> -static int early_init(void)
> +static void rlimit_unlimit_nofile(void)
>  {
> +	struct rlimit new;
> +
> +	new.rlim_cur = kdat.sysctl_nr_open;
> +	new.rlim_max = kdat.sysctl_nr_open;
> +
> +	if (prlimit(getpid(), RLIMIT_NOFILE, &new, NULL)) {
> +		pr_perror("rlimit: Can't setup RLIMIT_NOFILE for self");
> +		return;
> +	} else
> +		pr_debug("rlimit: RLIMIT_NOFILE unlimited for self\n");
> +
> +	service_fd_rlim_cur = kdat.sysctl_nr_open;
> +}
> +
> +static int early_init(const char *cmd)
> +{
> +	static const char *nofile_cmds[] = {
> +		"swrk", "service",
> +		"dump", "restore",
> +	};
> +	size_t i;
> +
> +	/*
> +	 * Service fd engine implies that file descriptors
> +	 * used won't be borrowed by the rest of the code
> +	 * and default 1024 limit is not enough for high
> +	 * loaded test/containers. Thus use kdat engine
> +	 * to fetch current system level limit for numbers
> +	 * of files allowed to open up and lift up own
> +	 * limits.
> +	 *
> +	 * Note we have to do it before the service fd
> +	 * get inited and we dont exit with errors here
> +	 * because in worst scenario where clash of fd
> +	 * happen we simply exit with explicit error
> +	 * during real action stage.
> +	 *
> +	 * Same time raising limits cause kernel fdtable
> +	 * to bloat so we do this only on the @nofile_cmds:
> +	 *
> +	 *  - on dump criu needs additional files for sfd,
> +	 *    thus if container already has many files opened
> +	 *    we need to have at least not less space when
> +	 *    fetching fds from a target process,
> +	 *    on restore the same -- we need to be able to
> +	 *    allocate all files from images;
> +	 *
> +	 *  - swrk and servise obtain requests on the fly,
> +	 *    thus we don't know if dump/restore will be
> +	 *    there.
> +	 */
> +	for (i = 0; i < ARRAY_SIZE(nofile_cmds); i++) {
> +		if (strcmp(nofile_cmds[i], cmd))
> +			continue;
> +		if (!kerndat_files_stat(true))
> +			rlimit_unlimit_nofile();
> +		break;
> +	}
> +
>  	if (init_service_fd())
>  		return 1;
>  
> @@ -94,7 +153,7 @@ int main(int argc, char *argv[], char *envp[])
>  
>  	log_set_loglevel(opts.log_level);
>  
> -	if (early_init())
> +	if (early_init(argv[optind]))
>  		return -1;
>  
>  	if (!strcmp(argv[1], "swrk")) {
Cyrill Gorcunov April 5, 2019, 8:48 p.m.
On Fri, Apr 05, 2019 at 09:40:13PM +0100, Radostin Stoyanov wrote:
> ping
> 

It is been reworked, I resent the series today.