[03/12] soccr/tcp: Add _pause and _resume to library and use it

Submitted by Pavel Emelianov on Aug. 5, 2016, 2:59 p.m.

Details

Message ID 57A4A9D7.7050001@virtuozzo.com
State Accepted
Series "Introduce library for socket C/R"
Commit a32a7e65339f44d165484eacb5b9f9296e196e07
Headers show

Commit Message

Pavel Emelianov Aug. 5, 2016, 2:59 p.m.
The calls put socket into a mode where it can be saved or
restored. Add take one out of this mode.

TODO: In criu we turn repair off (resume the socket) from
restorer blob that cannot use the library. Need to somehow
fix it.

Signed-off-by: Pavel Emelyanov <xemul@virtuozzo.com>
---
 criu/include/sk-inet.h |  2 ++
 criu/sk-tcp.c          | 18 ++++++++++++------
 soccr/soccr.c          | 49 +++++++++++++++++++++++++++++++++++++++++++++++++
 soccr/soccr.h          |  5 +++++
 4 files changed, 68 insertions(+), 6 deletions(-)

Patch hide | download patch | download mbox

diff --git a/criu/include/sk-inet.h b/criu/include/sk-inet.h
index 9d2bda6..772a663 100644
--- a/criu/include/sk-inet.h
+++ b/criu/include/sk-inet.h
@@ -33,6 +33,8 @@  struct inet_sk_desc {
 	int rfd;
 	int cpt_reuseaddr;
 	struct list_head rlist;
+
+	void *priv;
 };
 
 struct inet_port;
diff --git a/criu/sk-tcp.c b/criu/sk-tcp.c
index 8168138..a13954b 100644
--- a/criu/sk-tcp.c
+++ b/criu/sk-tcp.c
@@ -138,6 +138,7 @@  static int refresh_inet_sk(struct inet_sk_desc *sk, struct tcp_info *ti)
 static int tcp_repair_establised(int fd, struct inet_sk_desc *sk)
 {
 	int ret;
+	struct libsoccr_sk *socr;
 
 	pr_info("\tTurning repair on for socket %x\n", sk->sd.ino);
 	/*
@@ -157,10 +158,11 @@  static int tcp_repair_establised(int fd, struct inet_sk_desc *sk)
 			goto err2;
 	}
 
-	ret = tcp_repair_on(sk->rfd);
-	if (ret < 0)
+	socr = libsoccr_pause(sk->rfd);
+	if (!socr)
 		goto err3;
 
+	sk->priv = socr;
 	list_add_tail(&sk->rlist, &cpt_tcp_repair_sockets);
 	return 0;
 
@@ -185,7 +187,8 @@  static void tcp_unlock_one(struct inet_sk_desc *sk)
 			pr_perror("Failed to unlock TCP connection");
 	}
 
-	tcp_repair_off(sk->rfd);
+	libsoccr_resume(sk->priv);
+	sk->priv = NULL;
 
 	/*
 	 * tcp_repair_off modifies SO_REUSEADDR so
@@ -680,7 +683,7 @@  static int restore_tcp_window(int sk, TcpStreamEntry *tse)
 	return 0;
 }
 
-static int restore_tcp_conn_state(int sk, struct inet_sk_info *ii)
+static int restore_tcp_conn_state(int sk, struct libsoccr_sk *socr, struct inet_sk_info *ii)
 {
 	int aux;
 	struct cr_img *img;
@@ -770,12 +773,15 @@  int prepare_tcp_socks(struct task_restore_args *ta)
 
 int restore_one_tcp(int fd, struct inet_sk_info *ii)
 {
+	struct libsoccr_sk *sk;
+
 	pr_info("Restoring TCP connection\n");
 
-	if (tcp_repair_on(fd))
+	sk = libsoccr_pause(fd);
+	if (!sk)
 		return -1;
 
-	if (restore_tcp_conn_state(fd, ii))
+	if (restore_tcp_conn_state(fd, sk, ii))
 		return -1;
 
 	return 0;
diff --git a/soccr/soccr.c b/soccr/soccr.c
index 98b8f47..36b3887 100644
--- a/soccr/soccr.c
+++ b/soccr/soccr.c
@@ -1,3 +1,5 @@ 
+#include <netinet/tcp.h>
+#include <stdlib.h>
 #include "soccr.h"
 
 static void (*log)(unsigned int loglevel, const char *format, ...)
@@ -12,3 +14,50 @@  void libsoccr_set_log(unsigned int level, void (*fn)(unsigned int level, const c
 
 #define loge(msg, ...) do { if (log && (log_level >= SOCCR_LOG_ERR)) log(SOCCR_LOG_ERR, msg, ##__VA_ARGS__); } while (0)
 #define logd(msg, ...) do { if (log && (log_level >= SOCCR_LOG_DBG)) log(SOCCR_LOG_DBG, msg, ##__VA_ARGS__); } while (0)
+
+static int tcp_repair_on(int fd)
+{
+	int ret, aux = 1;
+
+	ret = setsockopt(fd, SOL_TCP, TCP_REPAIR, &aux, sizeof(aux));
+	if (ret < 0)
+		loge("Can't turn TCP repair mode ON");
+
+	return ret;
+}
+
+static void tcp_repair_off(int fd)
+{
+	int aux = 0, ret;
+
+	ret = setsockopt(fd, SOL_TCP, TCP_REPAIR, &aux, sizeof(aux));
+	if (ret < 0)
+		loge("Failed to turn off repair mode on socket: %m\n");
+}
+
+struct libsoccr_sk {
+	int fd;
+};
+
+struct libsoccr_sk *libsoccr_pause(int fd)
+{
+	struct libsoccr_sk *ret;
+
+	ret = malloc(sizeof(*ret));
+	if (!ret)
+		return NULL;
+
+	if (tcp_repair_on(fd) < 0) {
+		free(ret);
+		return NULL;
+	}
+
+	ret->fd = fd;
+	return ret;
+}
+
+void libsoccr_resume(struct libsoccr_sk *sk)
+{
+	tcp_repair_off(sk->fd);
+	free(sk);
+}
diff --git a/soccr/soccr.h b/soccr/soccr.h
index 07d7947..f5bdd18 100644
--- a/soccr/soccr.h
+++ b/soccr/soccr.h
@@ -9,4 +9,9 @@  void libsoccr_set_log(unsigned int level, void (*fn)(unsigned int level, const c
 #define SOCCR_LOG_ERR	1
 #define SOCCR_LOG_DBG	2
 
+struct libsoccr_sk;
+
+struct libsoccr_sk *libsoccr_pause(int fd);
+void libsoccr_resume(struct libsoccr_sk *sk);
+
 #endif

Comments

Dmitry Safonov Aug. 5, 2016, 3:43 p.m.
On 08/05/2016 05:59 PM, Pavel Emelyanov wrote:
> The calls put socket into a mode where it can be saved or
> restored. Add take one out of this mode.
>
> TODO: In criu we turn repair off (resume the socket) from
> restorer blob that cannot use the library. Need to somehow
> fix it.
>
> Signed-off-by: Pavel Emelyanov <xemul@virtuozzo.com>

So, if we generate _static_ library for now -- is there an issue, that
prevents compiling it into restorer object blob? The size of library?

Or you prefer to make the library shared firstly and then to think what 
to do with dlopen() or how to compile it into the blob? What's the plan?
Or I didn't understand the issue correctly :)
Pavel Emelianov Aug. 5, 2016, 3:59 p.m.
On 08/05/2016 06:43 PM, Dmitry Safonov wrote:
> On 08/05/2016 05:59 PM, Pavel Emelyanov wrote:
>> The calls put socket into a mode where it can be saved or
>> restored. Add take one out of this mode.
>>
>> TODO: In criu we turn repair off (resume the socket) from
>> restorer blob that cannot use the library. Need to somehow
>> fix it.
>>
>> Signed-off-by: Pavel Emelyanov <xemul@virtuozzo.com>
> 
> So, if we generate _static_ library for now -- is there an issue, that
> prevents compiling it into restorer object blob? The size of library?

Absence of glibc in the restorer blob :)

> Or you prefer to make the library shared firstly and then to think what 
> to do with dlopen() or how to compile it into the blob? What's the plan?
> Or I didn't understand the issue correctly :)

The issue is that in restorer there's a call to setsocopt(sk, TCP_REPAIR, 0)
which is equals to libsoccr's tcp_repair_off() routine.

This could be solved by linking libsoccr into restorer, but we don't have
glibc there and thus libsoccr would have to be equipped with #ifdefs to
help using either glibc's setsocktopt or sys_setsockopt.

And the latter is overkill for a single call. All the more so the library
is not for internal criu usage, I will make it .so soon to let other people
play with socket C/R only.

-- Pavel
Dmitry Safonov Aug. 5, 2016, 4 p.m.
On 08/05/2016 06:59 PM, Pavel Emelyanov wrote:
> On 08/05/2016 06:43 PM, Dmitry Safonov wrote:
>> On 08/05/2016 05:59 PM, Pavel Emelyanov wrote:
>>> The calls put socket into a mode where it can be saved or
>>> restored. Add take one out of this mode.
>>>
>>> TODO: In criu we turn repair off (resume the socket) from
>>> restorer blob that cannot use the library. Need to somehow
>>> fix it.
>>>
>>> Signed-off-by: Pavel Emelyanov <xemul@virtuozzo.com>
>>
>> So, if we generate _static_ library for now -- is there an issue, that
>> prevents compiling it into restorer object blob? The size of library?
>
> Absence of glibc in the restorer blob :)
>
>> Or you prefer to make the library shared firstly and then to think what
>> to do with dlopen() or how to compile it into the blob? What's the plan?
>> Or I didn't understand the issue correctly :)
>
> The issue is that in restorer there's a call to setsocopt(sk, TCP_REPAIR, 0)
> which is equals to libsoccr's tcp_repair_off() routine.
>
> This could be solved by linking libsoccr into restorer, but we don't have
> glibc there and thus libsoccr would have to be equipped with #ifdefs to
> help using either glibc's setsocktopt or sys_setsockopt.
>
> And the latter is overkill for a single call. All the more so the library
> is not for internal criu usage, I will make it .so soon to let other people
> play with socket C/R only.

Oh, now I understand, thanks on explanations :)