[7/7] tty: ctty -- Add support for multiple inherited ctty opened

Submitted by Cyrill Gorcunov on Feb. 15, 2018, 8:30 p.m.

Details

Message ID 20180215203030.7513-8-gorcunov@virtuozzo.com
State New
Series "criu: Add support for ctty inheritance"
Headers show

Commit Message

Cyrill Gorcunov Feb. 15, 2018, 8:30 p.m.
When kernel does fork a process it gets reference to that named
current tty which stands for controlling terminal of a session
(if present). Because we unable to propagate such terminals
due to restore procedure specifics (we fork processes first,
then restore the terminals on demand setting up session if
needed) we refused to checkpoint the containers where ctty
is opened in one of children of a session leader.

Now if ve.ctty interface is present in our vz kernel we
can proceed checkpoint and restore the container then.

Basically the idea is following:

 - use process sid when dumping opened /dev/tty so that
   different processes have own records in tty-info.img
   (previously only one ctty record is kept inside, shared
    for current terminals)

 - on restore collect these tty-info recods, find the session
   leader process and queue all opened ctty with same sid on
   this leader peer

 - then we allocate special CTL_TTY_OFF service fd and wait
   until it opened (all queued ctty are waiting for this fd
   to become "restored")

 - when we're setting up a session via controlling terminal
   we pass the kernel a recod of pids (which processes should
   have current terminal propagated) via ve.ctty interface,
   and we kick all queued ctty waiters to proceed opening,
   because now open ctty is valid

https://jira.sw.ru/browse/PSBM-76490

Signed-off-by: Cyrill Gorcunov <gorcunov@virtuozzo.com>
---
 criu/tty.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 178 insertions(+), 14 deletions(-)

Patch hide | download patch | download mbox

diff --git a/criu/tty.c b/criu/tty.c
index 41d8dcaa2..7d6dc63c3 100644
--- a/criu/tty.c
+++ b/criu/tty.c
@@ -31,6 +31,7 @@ 
 #include "action-scripts.h"
 #include "filesystems.h"
 #include "mount.h"
+#include "kerndat.h"
 
 #include "protobuf.h"
 #include "util.h"
@@ -98,6 +99,9 @@  struct tty_info {
 	bool				create;
 	bool				inherit;
 
+	struct pstree_item		*session_leader;
+	struct list_head		session_waiters;
+
 	struct tty_info			*ctl_tty;
 	struct tty_info			*link;
 	struct tty_data_entry		*tty_data;
@@ -122,6 +126,8 @@  struct tty_dump_info {
 	struct tty_dump_info		*link;
 	void				*tty_data;
 	size_t				tty_data_size;
+
+	unsigned int			ctty_index;
 };
 
 static bool stdin_isatty = false;
@@ -133,11 +139,11 @@  static LIST_HEAD(all_ttys);
  * If this won't be enough in future we simply need to
  * change tracking mechanism to some more extendable.
  *
- * This particular bitmap requires 256 bytes of memory.
+ * This particular bitmap requires 512 bytes of memory.
  * Pretty acceptable trade off in a sake of simplicity.
  */
 
-#define MAX_TTYS	1024
+#define MAX_TTYS	2048
 
 /*
  * Custom indices should be even numbers just in case if we
@@ -145,11 +151,16 @@  static LIST_HEAD(all_ttys);
  */
 
 #define MAX_PTY_INDEX	1000
+
 #define CONSOLE_INDEX	1002
 #define VT_INDEX	1004
 #define CTTY_INDEX	1006
 #define ETTY_INDEX	1008
 #define STTY_INDEX	1010
+
+#define MIN_CTTY_INDEX	1010
+#define MAX_CTTY_INDEX	2002
+
 #define INDEX_ERR	(MAX_TTYS + 1)
 
 struct tty_driver {
@@ -251,10 +262,44 @@  static struct tty_driver console_driver = {
 	.open			= open_simple_tty,
 };
 
+static int ctty_fd_get_index(int fd, const struct fd_parms *p)
+{
+	static unsigned int next_index = 0;
+	struct tty_dump_info *dinfo;
+
+	if (!p->virt_sid) {
+		pr_err("Virt sid not provided\n");
+		return INDEX_ERR;
+	}
+
+	list_for_each_entry(dinfo, &all_ttys, list) {
+		if (dinfo->driver->type != TTY_TYPE__CTTY)
+			continue;
+		if (dinfo->sid == p->virt_sid)
+			return dinfo->index;
+	}
+
+	if (next_index < MAX_CTTY_INDEX)
+		return MIN_CTTY_INDEX + next_index++;
+
+	pr_err("Index for /dev/tty is too big\n");
+	return INDEX_ERR;
+}
+
+static int ctty_img_get_index(struct tty_info *ti)
+{
+	/*
+	 * On restore we don't care about index, because
+	 * sessions restore goes via pty driver.
+	 */
+	return CTTY_INDEX;
+}
+
 static struct tty_driver ctty_driver = {
 	.type			= TTY_TYPE__CTTY,
 	.name			= "ctty",
-	.index			= CTTY_INDEX,
+	.fd_get_index		= ctty_fd_get_index,
+	.img_get_index		= ctty_img_get_index,
 	.open			= open_simple_tty,
 };
 
@@ -758,6 +803,80 @@  static int tty_set_prgp(int fd, int group)
 	return 0;
 }
 
+static int do_tty_propagate_session(void *arg, int fd, int pid)
+{
+	static const char path[] = "/sys/fs/cgroup/ve/ve.ctty";
+	char *res = arg;
+	int ret;
+
+	fd = open(path, O_RDWR);
+	if (fd < 0) {
+		pr_perror("Can't open %s", path);
+		return -1;
+	}
+
+	ret = write(fd, res, strlen(res) + 1);
+	if (ret != (strlen(res) + 1)) {
+		pr_perror("Can't write %s", res);
+	} else {
+		pr_debug("session propagated %s\n", res);
+		ret = 0;
+	}
+	close(fd);
+	return ret;
+}
+
+static int tty_propagate_session(struct tty_info *info)
+{
+	struct pstree_item *leader, *item;
+	struct tty_info *peer;
+	char *res = NULL;
+	int ret = 0;
+
+	if (!info->session_leader)
+		return 0;
+
+	if (!kdat.ve_can_inherit_ctty) {
+		pr_warn_once("ve.ctty interface is not present, restore may hang\n");
+		return 0;
+	}
+
+	leader = info->session_leader;
+
+	for_each_pstree_item(item) {
+		if (leader != item && item->sid == leader->sid) {
+			res = xstrcat(res, " %d", item->pid->real);
+			if (!res)
+				return -1;
+		}
+	}
+
+	if (res) {
+		char *_res = res;
+
+		res = xstrcat(NULL, "%d%s", leader->pid->real, res);
+		if (!res) {
+			xfree(_res);
+			return -1;
+		}
+
+		pr_debug("Propagating session via VE interface %s\n", res);
+		ret = userns_call(do_tty_propagate_session, 0, res, strlen(res) + 1, -1);
+		xfree(res);
+
+		if (ret)
+			return ret;
+	}
+
+	list_for_each_entry(peer, &info->session_waiters, session_waiters) {
+		struct fdinfo_list_entry *fle = file_master(&peer->d);
+		pr_debug("\tkicking process %d\n", fle->pid);
+		set_fds_event(fle->pid);
+	}
+
+	return 0;
+}
+
 int tty_restore_ctl_terminal(struct file_desc *d, int fd)
 {
 	struct tty_info *info = container_of(d, struct tty_info, d);
@@ -801,6 +920,8 @@  int tty_restore_ctl_terminal(struct file_desc *d, int fd)
 	ret = tty_set_sid(slave);
 	if (!ret)
 		ret = tty_set_prgp(slave, info->tie->pgrp);
+	if (!ret)
+		ret = tty_propagate_session(info);
 
 	close(slave);
 err:
@@ -1229,13 +1350,28 @@  static bool tty_deps_restored(struct tty_info *info)
 	struct tty_info *tmp;
 
 	if (info->driver->type == TTY_TYPE__CTTY) {
-		list_for_each_entry(fle, list, ps_list) {
-			if (fle->desc->ops->type != FD_TYPES__TTY || fle->desc == &info->d)
-				continue;
-
-			/* ctty needs all others are restored */
-			if (fle->stage != FLE_RESTORED)
-				return false;
+		/*
+		 * If session has been inherited via fork(),
+		 * we should wait until it's propagated.
+		 */
+		if (info->session_leader) {
+			list_for_each_entry(fle, &rsti(info->session_leader)->fds, ps_list) {
+				if (fle->desc->ops->type != FD_TYPES__TTY ||
+				    fle->fe->fd != get_service_fd(CTL_TTY_OFF))
+					continue;
+
+				if (fle->stage != FLE_RESTORED)
+					return false;
+			}
+		} else {
+			list_for_each_entry(fle, list, ps_list) {
+				if (fle->desc->ops->type != FD_TYPES__TTY || fle->desc == &info->d)
+					continue;
+
+				/* ctty needs all others are restored */
+				if (fle->stage != FLE_RESTORED)
+					return false;
+			}
 		}
 	} else if (!tty_is_master(info)) {
 		list_for_each_entry(fle, list, ps_list) {
@@ -1307,6 +1443,7 @@  static struct pstree_item *find_session_leader(pid_t sid)
 static int tty_find_restoring_task(struct tty_info *info)
 {
 	struct pstree_item *item;
+	struct tty_info *peer;
 
 	/*
 	 * The overall scenario is the following (note
@@ -1385,6 +1522,26 @@  static int tty_find_restoring_task(struct tty_info *info)
 		 */
 		item = find_session_leader(info->tie->sid);
 		if (item) {
+			info->session_leader = item;
+			/*
+			 * Collect terminals which gonna wait for
+			 * session to be restored and propagated
+			 */
+			list_for_each_entry(peer, &all_ttys, list) {
+				if (peer->driver->type != TTY_TYPE__CTTY ||
+				    peer->tie->sid != info->tie->sid ||
+				    peer == info)
+					continue;
+
+				BUG_ON(peer->session_leader);
+
+				peer->session_leader = item;
+				list_add(&peer->session_waiters,
+					 &info->session_waiters);
+				pr_debug("Collect waiter %#x/%d to %#x/%d\n",
+					 peer->tfe->id, peer->tie->sid,
+					 info->tfe->id, info->tie->sid);
+			}
 			pr_info("Set a control terminal %#x to %d\n",
 				info->tfe->id, info->tie->sid);
 			return prepare_ctl_tty(vpid(item),
@@ -1733,6 +1890,9 @@  static int tty_info_setup(struct tty_info *info)
 	info->tty_data = NULL;
 	info->link = NULL;
 
+	INIT_LIST_HEAD(&info->session_waiters);
+	info->session_leader = NULL;
+
 	/*
 	 * The image might have no reg file record in old CRIU, so
 	 * lets don't fail for a while. After a couple of releases
@@ -1917,6 +2077,8 @@  static int dump_tty_info(int lfd, u32 id, const struct fd_parms *p, int mnt_id,
 			return -1;
 		}
 		dinfo->index	= index;
+	} else if (driver->type == TTY_TYPE__CTTY) {
+		dinfo->index	= index;
 	} else {
 		dinfo->index	= -1;
 		dinfo->lfd	= -1;
@@ -2323,10 +2485,12 @@  static int tty_verify_ctty(void)
 			       d->sid);
 			return -ENOENT;
 		} else if (n->pid_real != d->pid_real) {
-			pr_err("ctty inheritance detected sid %d "
-			       "(ctty pid_real %d pty pid_real %d)\n",
-			       d->sid, d->pid_real, n->pid_real);
-			return -ENOENT;
+			if (!kdat.ve_can_inherit_ctty) {
+				pr_err("ctty inheritance detected sid %d "
+				       "(ctty pid_real %d pty pid_real %d)\n",
+				       d->sid, d->pid_real, n->pid_real);
+				return -ENOENT;
+			}
 		}
 	}