[RHEL7,COMMIT] net/sctp: Allocate SSN map on a per-page basis

Submitted by Konstantin Khorenko on Aug. 20, 2018, 2:14 p.m.

Details

Message ID 201808201414.w7KEE9Fp002719@finist_ce7.work
State New
Series "net/sctp: Allocate SSN map on a per-page basis"
Headers show

Commit Message

Konstantin Khorenko Aug. 20, 2018, 2:14 p.m.
The commit is pushed to "branch-rh7-3.10.0-862.11.6.vz7.64.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh7-3.10.0-862.11.6.vz7.64.5
------>
commit 9a125c47fe764b63803e20a74a81db0b64230765
Author: Oleg Babin <obabin@virtuozzo.com>
Date:   Mon Aug 20 17:14:09 2018 +0300

    net/sctp: Allocate SSN map on a per-page basis
    
    SCTP protocol allocates TCB on receiving INIT and COOKIE ECHO chunks
    which specify input and output stream count. As the total count of
    those streams can be up to (2^16 - 1) of each type, it is possible
    to occupy a fifth order of memory for only one type of streams. As
    allocation can happen in a softirq contex with GFP_ATOMIC flag set
    and we actually do not need the memory to be physically contiguous,
    for performance reasons we use flex_array which allocates memory on
    a per-page basis.
    
    v2: Use flex_array instead of allocating pages manually.
    
    https://jira.sw.ru/browse/PSBM-82552
    
    Signed-off-by: Oleg Babin <obabin@virtuozzo.com>
---
 include/net/sctp/structs.h | 26 ++++++++----
 net/sctp/ssnmap.c          | 99 ++++++++++++++++++++++------------------------
 2 files changed, 65 insertions(+), 60 deletions(-)

Patch hide | download patch | download mbox

diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
index 6bcd9d18b751..432affdacd7a 100644
--- a/include/net/sctp/structs.h
+++ b/include/net/sctp/structs.h
@@ -64,6 +64,7 @@ 
 #include <linux/atomic.h>		/* This gets us atomic counters.  */
 #include <linux/skbuff.h>	/* We need sk_buff_head. */
 #include <linux/workqueue.h>	/* We need tq_struct.	 */
+#include <linux/flex_array.h>	/* We need flex_array.   */
 #include <linux/sctp.h>		/* We need sctp* header structs.  */
 #include <net/sctp/auth.h>	/* We need auth specific structs */
 #include <net/ip.h>		/* For inet_skb_parm */
@@ -396,10 +397,17 @@  typedef struct sctp_sender_hb_info {
  *
  *  This is the structure we use to track both our outbound and inbound
  *  SSN, or Stream Sequence Numbers.
+ *
+ *  As the total count of input or output streams can be up to (2^16 - 1)
+ *  of each type, it is possible to occupy a fifth order of memory for
+ *  only one type of streams. As allocation can happen in a softirq
+ *  contex with GFP_ATOMIC flag set and we actually do not need the memory
+ *  to be physically contiguous, for performance reasons we use flex_array
+ *  which allocates memory on a per-page basis.
  */
 
 struct sctp_stream {
-	__u16 *ssn;
+	struct flex_array *ssn;
 	unsigned int len;
 };
 
@@ -408,30 +416,32 @@  struct sctp_ssnmap {
 	struct sctp_stream out;
 };
 
-struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
-				    gfp_t gfp);
+struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp);
 void sctp_ssnmap_free(struct sctp_ssnmap *map);
 void sctp_ssnmap_clear(struct sctp_ssnmap *map);
 
 /* What is the current SSN number for this stream? */
 static inline __u16 sctp_ssn_peek(struct sctp_stream *stream, __u16 id)
 {
-	return stream->ssn[id];
+	__u16 *ssnp = flex_array_get(stream->ssn, id);
+	return *ssnp;
 }
 
 /* Return the next SSN number for this stream.	*/
 static inline __u16 sctp_ssn_next(struct sctp_stream *stream, __u16 id)
 {
-	return stream->ssn[id]++;
+	__u16 *ssnp = flex_array_get(stream->ssn, id);
+	return (*ssnp)++;
 }
 
 /* Skip over this ssn and all below. */
-static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id, 
+static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id,
 				 __u16 ssn)
 {
-	stream->ssn[id] = ssn+1;
+	__u16 *ssnp = flex_array_get(stream->ssn, id);
+	*ssnp = ssn + 1;
 }
-              
+
 /*
  * Pointers to address related SCTP functions.
  * (i.e. things that depend on the address family.)
diff --git a/net/sctp/ssnmap.c b/net/sctp/ssnmap.c
index da8603523808..06a6e46c6d1e 100644
--- a/net/sctp/ssnmap.c
+++ b/net/sctp/ssnmap.c
@@ -41,37 +41,57 @@ 
 #include <net/sctp/sctp.h>
 #include <net/sctp/sm.h>
 
-static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
-					    __u16 out);
+/* Allocate memory pages for one type of stream (in or out). */
+static int sctp_stream_alloc(struct sctp_stream *stream, __u16 len, gfp_t gfp)
+{
+	int err = -ENOMEM;
+
+	stream->ssn = flex_array_alloc(sizeof(__u16), len, gfp);
+	if (stream->ssn) {
+		err = flex_array_prealloc(stream->ssn, 0, len, gfp);
+		if (err) {
+			flex_array_free(stream->ssn);
+			stream->ssn = NULL;
+		}
+	}
+
+	return err;
+}
 
-/* Storage size needed for map includes 2 headers and then the
- * specific needs of in or out streams.
- */
-static inline size_t sctp_ssnmap_size(__u16 in, __u16 out)
+/* Free memory pages for one type of stream (in or out). */
+static void sctp_stream_free(struct sctp_stream *stream)
 {
-	return sizeof(struct sctp_ssnmap) + (in + out) * sizeof(__u16);
+	if (stream->ssn)
+		flex_array_free(stream->ssn);
 }
 
+/* Clear all SSNs for one type of stream (in or out). */
+static void sctp_stream_clear(struct sctp_stream *stream)
+{
+	unsigned int i;
+
+	for (i = 0; i < stream->len; i++)
+		flex_array_clear(stream->ssn, i);
+}
 
 /* Create a new sctp_ssnmap.
- * Allocate room to store at least 'len' contiguous TSNs.
+ * Allocate room to store at least 'in' + 'out' SSNs.
  */
-struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
-				    gfp_t gfp)
+struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp)
 {
 	struct sctp_ssnmap *retval;
-	int size;
-
-	size = sctp_ssnmap_size(in, out);
-	if (size <= KMALLOC_MAX_SIZE)
-		retval = kmalloc(size, gfp);
-	else
-		retval = (struct sctp_ssnmap *)
-			  __get_free_pages(gfp, get_order(size));
+	int err;
+
+	retval = (struct sctp_ssnmap *)kzalloc(sizeof(struct sctp_ssnmap), gfp);
 	if (!retval)
 		goto fail;
 
-	if (!sctp_ssnmap_init(retval, in, out))
+	err = sctp_stream_alloc(&retval->in, in, gfp);
+	if (err)
+		goto fail_map;
+
+	err = sctp_stream_alloc(&retval->out, out, gfp);
+	if (err)
 		goto fail_map;
 
 	SCTP_DBG_OBJCNT_INC(ssnmap);
@@ -79,54 +99,29 @@  struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
 	return retval;
 
 fail_map:
-	if (size <= KMALLOC_MAX_SIZE)
-		kfree(retval);
-	else
-		free_pages((unsigned long)retval, get_order(size));
+	sctp_stream_free(&retval->in);
+	sctp_stream_free(&retval->out);
+	kfree(retval);
 fail:
 	return NULL;
 }
 
-
-/* Initialize a block of memory as a ssnmap.  */
-static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
-					    __u16 out)
-{
-	memset(map, 0x00, sctp_ssnmap_size(in, out));
-
-	/* Start 'in' stream just after the map header. */
-	map->in.ssn = (__u16 *)&map[1];
-	map->in.len = in;
-
-	/* Start 'out' stream just after 'in'. */
-	map->out.ssn = &map->in.ssn[in];
-	map->out.len = out;
-
-	return map;
-}
-
 /* Clear out the ssnmap streams.  */
 void sctp_ssnmap_clear(struct sctp_ssnmap *map)
 {
-	size_t size;
-
-	size = (map->in.len + map->out.len) * sizeof(__u16);
-	memset(map->in.ssn, 0x00, size);
+	sctp_stream_clear(&map->in);
+	sctp_stream_clear(&map->out);
 }
 
 /* Dispose of a ssnmap.  */
 void sctp_ssnmap_free(struct sctp_ssnmap *map)
 {
-	int size;
-
 	if (unlikely(!map))
 		return;
 
-	size = sctp_ssnmap_size(map->in.len, map->out.len);
-	if (size <= KMALLOC_MAX_SIZE)
-		kfree(map);
-	else
-		free_pages((unsigned long)map, get_order(size));
+	sctp_stream_free(&map->in);
+	sctp_stream_free(&map->out);
+	kfree(map);
 
 	SCTP_DBG_OBJCNT_DEC(ssnmap);
 }

Comments

Konstantin Khorenko Aug. 20, 2018, 2:19 p.m.
Please ignore.

--
Best regards,

Konstantin Khorenko,
Virtuozzo Linux Kernel Team

On 08/20/2018 05:14 PM, Konstantin Khorenko wrote:
> The commit is pushed to "branch-rh7-3.10.0-862.11.6.vz7.64.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
> after rh7-3.10.0-862.11.6.vz7.64.5
> ------>
> commit 9a125c47fe764b63803e20a74a81db0b64230765
> Author: Oleg Babin <obabin@virtuozzo.com>
> Date:   Mon Aug 20 17:14:09 2018 +0300
>
>     net/sctp: Allocate SSN map on a per-page basis
>
>     SCTP protocol allocates TCB on receiving INIT and COOKIE ECHO chunks
>     which specify input and output stream count. As the total count of
>     those streams can be up to (2^16 - 1) of each type, it is possible
>     to occupy a fifth order of memory for only one type of streams. As
>     allocation can happen in a softirq contex with GFP_ATOMIC flag set
>     and we actually do not need the memory to be physically contiguous,
>     for performance reasons we use flex_array which allocates memory on
>     a per-page basis.
>
>     v2: Use flex_array instead of allocating pages manually.
>
>     https://jira.sw.ru/browse/PSBM-82552
>
>     Signed-off-by: Oleg Babin <obabin@virtuozzo.com>
> ---
>  include/net/sctp/structs.h | 26 ++++++++----
>  net/sctp/ssnmap.c          | 99 ++++++++++++++++++++++------------------------
>  2 files changed, 65 insertions(+), 60 deletions(-)
>
> diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
> index 6bcd9d18b751..432affdacd7a 100644
> --- a/include/net/sctp/structs.h
> +++ b/include/net/sctp/structs.h
> @@ -64,6 +64,7 @@
>  #include <linux/atomic.h>		/* This gets us atomic counters.  */
>  #include <linux/skbuff.h>	/* We need sk_buff_head. */
>  #include <linux/workqueue.h>	/* We need tq_struct.	 */
> +#include <linux/flex_array.h>	/* We need flex_array.   */
>  #include <linux/sctp.h>		/* We need sctp* header structs.  */
>  #include <net/sctp/auth.h>	/* We need auth specific structs */
>  #include <net/ip.h>		/* For inet_skb_parm */
> @@ -396,10 +397,17 @@ typedef struct sctp_sender_hb_info {
>   *
>   *  This is the structure we use to track both our outbound and inbound
>   *  SSN, or Stream Sequence Numbers.
> + *
> + *  As the total count of input or output streams can be up to (2^16 - 1)
> + *  of each type, it is possible to occupy a fifth order of memory for
> + *  only one type of streams. As allocation can happen in a softirq
> + *  contex with GFP_ATOMIC flag set and we actually do not need the memory
> + *  to be physically contiguous, for performance reasons we use flex_array
> + *  which allocates memory on a per-page basis.
>   */
>
>  struct sctp_stream {
> -	__u16 *ssn;
> +	struct flex_array *ssn;
>  	unsigned int len;
>  };
>
> @@ -408,30 +416,32 @@ struct sctp_ssnmap {
>  	struct sctp_stream out;
>  };
>
> -struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> -				    gfp_t gfp);
> +struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp);
>  void sctp_ssnmap_free(struct sctp_ssnmap *map);
>  void sctp_ssnmap_clear(struct sctp_ssnmap *map);
>
>  /* What is the current SSN number for this stream? */
>  static inline __u16 sctp_ssn_peek(struct sctp_stream *stream, __u16 id)
>  {
> -	return stream->ssn[id];
> +	__u16 *ssnp = flex_array_get(stream->ssn, id);
> +	return *ssnp;
>  }
>
>  /* Return the next SSN number for this stream.	*/
>  static inline __u16 sctp_ssn_next(struct sctp_stream *stream, __u16 id)
>  {
> -	return stream->ssn[id]++;
> +	__u16 *ssnp = flex_array_get(stream->ssn, id);
> +	return (*ssnp)++;
>  }
>
>  /* Skip over this ssn and all below. */
> -static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id,
> +static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id,
>  				 __u16 ssn)
>  {
> -	stream->ssn[id] = ssn+1;
> +	__u16 *ssnp = flex_array_get(stream->ssn, id);
> +	*ssnp = ssn + 1;
>  }
> -
> +
>  /*
>   * Pointers to address related SCTP functions.
>   * (i.e. things that depend on the address family.)
> diff --git a/net/sctp/ssnmap.c b/net/sctp/ssnmap.c
> index da8603523808..06a6e46c6d1e 100644
> --- a/net/sctp/ssnmap.c
> +++ b/net/sctp/ssnmap.c
> @@ -41,37 +41,57 @@
>  #include <net/sctp/sctp.h>
>  #include <net/sctp/sm.h>
>
> -static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
> -					    __u16 out);
> +/* Allocate memory pages for one type of stream (in or out). */
> +static int sctp_stream_alloc(struct sctp_stream *stream, __u16 len, gfp_t gfp)
> +{
> +	int err = -ENOMEM;
> +
> +	stream->ssn = flex_array_alloc(sizeof(__u16), len, gfp);
> +	if (stream->ssn) {
> +		err = flex_array_prealloc(stream->ssn, 0, len, gfp);
> +		if (err) {
> +			flex_array_free(stream->ssn);
> +			stream->ssn = NULL;
> +		}
> +	}
> +
> +	return err;
> +}
>
> -/* Storage size needed for map includes 2 headers and then the
> - * specific needs of in or out streams.
> - */
> -static inline size_t sctp_ssnmap_size(__u16 in, __u16 out)
> +/* Free memory pages for one type of stream (in or out). */
> +static void sctp_stream_free(struct sctp_stream *stream)
>  {
> -	return sizeof(struct sctp_ssnmap) + (in + out) * sizeof(__u16);
> +	if (stream->ssn)
> +		flex_array_free(stream->ssn);
>  }
>
> +/* Clear all SSNs for one type of stream (in or out). */
> +static void sctp_stream_clear(struct sctp_stream *stream)
> +{
> +	unsigned int i;
> +
> +	for (i = 0; i < stream->len; i++)
> +		flex_array_clear(stream->ssn, i);
> +}
>
>  /* Create a new sctp_ssnmap.
> - * Allocate room to store at least 'len' contiguous TSNs.
> + * Allocate room to store at least 'in' + 'out' SSNs.
>   */
> -struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> -				    gfp_t gfp)
> +struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp)
>  {
>  	struct sctp_ssnmap *retval;
> -	int size;
> -
> -	size = sctp_ssnmap_size(in, out);
> -	if (size <= KMALLOC_MAX_SIZE)
> -		retval = kmalloc(size, gfp);
> -	else
> -		retval = (struct sctp_ssnmap *)
> -			  __get_free_pages(gfp, get_order(size));
> +	int err;
> +
> +	retval = (struct sctp_ssnmap *)kzalloc(sizeof(struct sctp_ssnmap), gfp);
>  	if (!retval)
>  		goto fail;
>
> -	if (!sctp_ssnmap_init(retval, in, out))
> +	err = sctp_stream_alloc(&retval->in, in, gfp);
> +	if (err)
> +		goto fail_map;
> +
> +	err = sctp_stream_alloc(&retval->out, out, gfp);
> +	if (err)
>  		goto fail_map;
>
>  	SCTP_DBG_OBJCNT_INC(ssnmap);
> @@ -79,54 +99,29 @@ struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
>  	return retval;
>
>  fail_map:
> -	if (size <= KMALLOC_MAX_SIZE)
> -		kfree(retval);
> -	else
> -		free_pages((unsigned long)retval, get_order(size));
> +	sctp_stream_free(&retval->in);
> +	sctp_stream_free(&retval->out);
> +	kfree(retval);
>  fail:
>  	return NULL;
>  }
>
> -
> -/* Initialize a block of memory as a ssnmap.  */
> -static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
> -					    __u16 out)
> -{
> -	memset(map, 0x00, sctp_ssnmap_size(in, out));
> -
> -	/* Start 'in' stream just after the map header. */
> -	map->in.ssn = (__u16 *)&map[1];
> -	map->in.len = in;
> -
> -	/* Start 'out' stream just after 'in'. */
> -	map->out.ssn = &map->in.ssn[in];
> -	map->out.len = out;
> -
> -	return map;
> -}
> -
>  /* Clear out the ssnmap streams.  */
>  void sctp_ssnmap_clear(struct sctp_ssnmap *map)
>  {
> -	size_t size;
> -
> -	size = (map->in.len + map->out.len) * sizeof(__u16);
> -	memset(map->in.ssn, 0x00, size);
> +	sctp_stream_clear(&map->in);
> +	sctp_stream_clear(&map->out);
>  }
>
>  /* Dispose of a ssnmap.  */
>  void sctp_ssnmap_free(struct sctp_ssnmap *map)
>  {
> -	int size;
> -
>  	if (unlikely(!map))
>  		return;
>
> -	size = sctp_ssnmap_size(map->in.len, map->out.len);
> -	if (size <= KMALLOC_MAX_SIZE)
> -		kfree(map);
> -	else
> -		free_pages((unsigned long)map, get_order(size));
> +	sctp_stream_free(&map->in);
> +	sctp_stream_free(&map->out);
> +	kfree(map);
>
>  	SCTP_DBG_OBJCNT_DEC(ssnmap);
>  }
> .
>