From patchwork Tue Apr 9 15:24:19 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: [v4,3/3] early_init: Raise nr_open limit on particular commands From: Cyrill Gorcunov X-Patchwork-Id: 10438 Message-Id: <20190409152419.GE1421@uranus.lan> To: Andrei Vagin Cc: CriuML Date: Tue, 9 Apr 2019 18:24:19 +0300 For swrk, service, dump, restore we need to to raise nr_file limit to be able to process containers with huge number of files opened. Signed-off-by: Cyrill Gorcunov --- v4: raise limits for restore as well criu/cr-restore.c | 32 ---------------------- criu/crtools.c | 69 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 34 deletions(-) diff --git a/criu/cr-restore.c b/criu/cr-restore.c index af2ca2921d00..a6cb8f1b3e07 100644 --- a/criu/cr-restore.c +++ b/criu/cr-restore.c @@ -2327,42 +2327,10 @@ int prepare_dummy_task_state(struct pstree_item *pi) return 0; } -static void rlimit_unlimit_nofile_self(void) -{ - struct rlimit new; - - new.rlim_cur = kdat.sysctl_nr_open; - new.rlim_max = kdat.sysctl_nr_open; - - if (prlimit(getpid(), RLIMIT_NOFILE, &new, NULL)) { - pr_perror("rlimir: Can't setup RLIMIT_NOFILE for self"); - return; - } else - pr_debug("rlimit: RLIMIT_NOFILE unlimited for self\n"); - service_fd_rlim_cur = kdat.sysctl_nr_open; -} - int cr_restore_tasks(void) { int ret = -1; - /* - * Service fd engine implies that file descriptors - * used won't be borrowed by the rest of the code - * and default 1024 limit is not enough for high - * loaded test/containers. Thus use kdat engine - * to fetch current system level limit for numbers - * of files allowed to open up and lift up own - * limits. - * - * Note we have to do it before the service fd - * get inited and we dont exit with errors here - * because in worst scenario where clash of fd - * happen we simply exit with explicit error - * during real action stage. - */ - rlimit_unlimit_nofile_self(); - if (cr_plugin_init(CR_PLUGIN_STAGE__RESTORE)) return -1; diff --git a/criu/crtools.c b/criu/crtools.c index 3f64000c1e87..95d221b483e8 100644 --- a/criu/crtools.c +++ b/criu/crtools.c @@ -51,8 +51,73 @@ #include "sysctl.h" #include "img-remote.h" -static int early_init(void) +static void rlimit_unlimit_nofile(void) { + struct rlimit new; + + new.rlim_cur = kdat.sysctl_nr_open; + new.rlim_max = kdat.sysctl_nr_open; + + if (prlimit(getpid(), RLIMIT_NOFILE, &new, NULL)) { + pr_perror("rlimit: Can't setup RLIMIT_NOFILE for self"); + return; + } else + pr_debug("rlimit: RLIMIT_NOFILE unlimited for self\n"); + + service_fd_rlim_cur = kdat.sysctl_nr_open; +} + +static int early_init(const char *cmd) +{ + static const char *nofile_cmds[] = { + "swrk", "service", + "dump", "pre-dump", + "restore", + }; + size_t i; + + /* + * Service fd engine implies that file descriptors + * used won't be borrowed by the rest of the code + * and default 1024 limit is not enough for high + * loaded test/containers. Thus use kdat engine + * to fetch current system level limit for numbers + * of files allowed to open up and lift up own + * limits. + * + * Note we have to do it before the service fd + * get inited and we dont exit with errors here + * because in worst scenario where clash of fd + * happen we simply exit with explicit error + * during real action stage. + * + * Same time raising limits cause kernel fdtable + * to bloat so we do this only on the @nofile_cmds: + * + * - on dump criu needs additional files for sfd, + * thus if container already has many files opened + * we need to have at least not less space when + * fetching fds from a target process; + * + * - on pre-dump we might need a lot of pipes to + * fetch huge number of pages to dump; + * + * - on restore we still need to raise limits since + * there is no guarantee that on dump we've not + * been hitting fd limit already; + * + * - swrk and service obtain requests on the fly, + * thus we don't know if on of above will be + * there thus raise limits. + */ + for (i = 0; i < ARRAY_SIZE(nofile_cmds); i++) { + if (strcmp(nofile_cmds[i], cmd)) + continue; + if (!kerndat_files_stat(true)) + rlimit_unlimit_nofile(); + break; + } + if (init_service_fd()) return 1; @@ -94,7 +159,7 @@ int main(int argc, char *argv[], char *envp[]) log_set_loglevel(opts.log_level); - if (early_init()) + if (early_init(argv[optind])) return -1; if (!strcmp(argv[1], "swrk")) {