[vz7] net/sctp: Allocate SSN map on a per-page basis

Submitted by Oleg Babin on March 27, 2018, 5:55 p.m.

Details

Message ID 1522173356-918090-1-git-send-email-obabin@virtuozzo.com
State New
Series "net/sctp: Allocate SSN map on a per-page basis"
Headers show

Commit Message

Oleg Babin March 27, 2018, 5:55 p.m.
SCTP protocol allocates TCB on receiving INIT and COOKIE ECHO chunks
which specify input and output stream count. As the total count of
those streams can be up to (2^16 - 1) of each type, it is possible
to occupy a fifth order of memory for only one type of streams. Plus
allocation can happen in a softirq contex with GFP_ATOMIC flag set
and we actually do not need the memory to be physically contiguous,
so for performance reasons we allocate memory on a per-page basis.

https://jira.sw.ru/browse/PSBM-82552

Signed-off-by: Oleg Babin <obabin@virtuozzo.com>
---
 include/net/sctp/structs.h |  44 +++++++++++---
 net/sctp/ssnmap.c          | 142 ++++++++++++++++++++++++++++-----------------
 2 files changed, 127 insertions(+), 59 deletions(-)

Patch hide | download patch | download mbox

diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
index c2f7064..0a09dbe 100644
--- a/include/net/sctp/structs.h
+++ b/include/net/sctp/structs.h
@@ -396,11 +396,29 @@  typedef struct sctp_sender_hb_info {
  *
  *  This is the structure we use to track both our outbound and inbound
  *  SSN, or Stream Sequence Numbers.
+ *
+ *  As the total count of input or output streams can be up to (2^16 - 1)
+ *  of each type, it is possible to occupy a fifth order of memory for
+ *  only one type of streams. Plus allocation can happen in a softirq
+ *  contex with GFP_ATOMIC flag set and we actually do not need the memory
+ *  to be physically contiguous, so for performance reasons we allocate
+ *  memory on a per-page basis.
  */
 
+#define SCTP_MAX_STREAMS	65535
+#define SCTP_SSN_PAGE_SHIFT	(PAGE_SHIFT - 1)  /* One SSN takes 2 bytes */
+#define SCTP_SSN_PAGE_SIZE	(1u << SCTP_SSN_PAGE_SHIFT)
+#define SCTP_SSN_INDEX_MASK	(SCTP_SSN_PAGE_SIZE - 1)
+#define SCTP_SSN_MAX_PAGES	((SCTP_MAX_STREAMS + SCTP_SSN_PAGE_SIZE - 1) \
+				 / SCTP_SSN_PAGE_SIZE)
+
+struct sctp_ssn_page {
+	__u16 ssn[SCTP_SSN_PAGE_SIZE];
+};
+
 struct sctp_stream {
-	__u16 *ssn;
 	unsigned int len;
+	struct sctp_ssn_page *pages[SCTP_SSN_MAX_PAGES];
 };
 
 struct sctp_ssnmap {
@@ -408,30 +426,42 @@  struct sctp_ssnmap {
 	struct sctp_stream out;
 };
 
-struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
-				    gfp_t gfp);
+struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp);
 void sctp_ssnmap_free(struct sctp_ssnmap *map);
 void sctp_ssnmap_clear(struct sctp_ssnmap *map);
 
+/* Helper function to get a pointer to a particular SSN storage. */
+static inline __u16 *sctp_ssn_ptr(struct sctp_stream *stream, __u16 id)
+{
+	unsigned int page = id >> SCTP_SSN_PAGE_SHIFT;
+	unsigned int index = id & SCTP_SSN_INDEX_MASK;
+
+	return &stream->pages[page]->ssn[index];
+}
+
 /* What is the current SSN number for this stream? */
 static inline __u16 sctp_ssn_peek(struct sctp_stream *stream, __u16 id)
 {
-	return stream->ssn[id];
+	__u16 *ssnp = sctp_ssn_ptr(stream, id);
+	return *ssnp;
 }
 
 /* Return the next SSN number for this stream.	*/
 static inline __u16 sctp_ssn_next(struct sctp_stream *stream, __u16 id)
 {
-	return stream->ssn[id]++;
+	__u16 *ssnp = sctp_ssn_ptr(stream, id);
+	return (*ssnp)++;
 }
 
 /* Skip over this ssn and all below. */
 static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id, 
 				 __u16 ssn)
 {
-	stream->ssn[id] = ssn+1;
+	__u16 *ssnp = sctp_ssn_ptr(stream, id);
+	*ssnp = ssn + 1;
+
 }
-              
+
 /*
  * Pointers to address related SCTP functions.
  * (i.e. things that depend on the address family.)
diff --git a/net/sctp/ssnmap.c b/net/sctp/ssnmap.c
index da86035..610ba6a 100644
--- a/net/sctp/ssnmap.c
+++ b/net/sctp/ssnmap.c
@@ -41,37 +41,100 @@ 
 #include <net/sctp/sctp.h>
 #include <net/sctp/sm.h>
 
-static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
-					    __u16 out);
+/* Allocate one memory page of SSN storage. */
+static inline int sctp_ssn_page_alloc(struct sctp_ssn_page **pagep, gfp_t gfp)
+{
+	BUILD_BUG_ON(sizeof(struct sctp_ssn_page) != PAGE_SIZE);
 
-/* Storage size needed for map includes 2 headers and then the
- * specific needs of in or out streams.
- */
-static inline size_t sctp_ssnmap_size(__u16 in, __u16 out)
+	*pagep = (struct sctp_ssn_page *)get_zeroed_page(gfp);
+	return *pagep != NULL;
+}
+
+/* Free one memory page of SSN storage. */
+static inline void sctp_ssn_page_free(struct sctp_ssn_page *page)
+{
+	if (page)
+		free_pages((unsigned long)page, 0);
+}
+
+/* Allocate memory for SSNs taking less than one memory page. */
+static inline int sctp_ssn_page_partial_alloc(struct sctp_ssn_page **pagep,
+					      unsigned int len, gfp_t gfp)
+{
+	if (len == 0)
+		return 1;
+
+	*pagep = (struct sctp_ssn_page *)kzalloc(len * sizeof(__u16), gfp);
+	return *pagep != NULL;
+}
+
+/* Free memory that is smaller than one memory page. */
+static inline void sctp_ssn_page_partial_free(struct sctp_ssn_page *page)
 {
-	return sizeof(struct sctp_ssnmap) + (in + out) * sizeof(__u16);
+	if (page)
+		kfree(page);
+}
+
+/* Allocate memory pages for one type of stream (in or out). */
+static int sctp_stream_alloc(struct sctp_stream *stream, __u16 len, gfp_t gfp)
+{
+	unsigned int pages = len >> SCTP_SSN_PAGE_SHIFT;
+	unsigned int rest = len & SCTP_SSN_INDEX_MASK;
+	unsigned int pi;
+
+	stream->len = len;
+
+	for (pi = 0; pi < pages; ++pi)
+		if (!sctp_ssn_page_alloc(&stream->pages[pi], gfp))
+			return 0;
+
+	if (!sctp_ssn_page_partial_alloc(&stream->pages[pi], rest, gfp))
+		return 0;
+
+	return 1;
 }
 
+/* Free memory pages for one type of stream (in or out). */
+static void sctp_stream_free(struct sctp_stream *stream)
+{
+	unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
+	unsigned int pi;
+
+	for (pi = 0; pi < pages; ++pi)
+		sctp_ssn_page_free(stream->pages[pi]);
+
+	sctp_ssn_page_partial_free(stream->pages[pi]);
+}
+
+/* Clear all SSNs for one type of stream (in or out). */
+static void sctp_stream_clear(struct sctp_stream *stream)
+{
+	unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
+	unsigned int rest = stream->len & SCTP_SSN_INDEX_MASK;
+	unsigned int pi;
+
+	for (pi = 0; pi < pages; ++pi)
+		memset(stream->pages[pi], 0, sizeof(struct sctp_ssn_page));
+
+	if (rest)
+		memset(stream->pages[pi], 0, rest * sizeof(__u16));
+}
 
 /* Create a new sctp_ssnmap.
- * Allocate room to store at least 'len' contiguous TSNs.
+ * Allocate room to store at least 'in' + 'out' SSNs.
  */
-struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
-				    gfp_t gfp)
+struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp)
 {
 	struct sctp_ssnmap *retval;
-	int size;
-
-	size = sctp_ssnmap_size(in, out);
-	if (size <= KMALLOC_MAX_SIZE)
-		retval = kmalloc(size, gfp);
-	else
-		retval = (struct sctp_ssnmap *)
-			  __get_free_pages(gfp, get_order(size));
+
+	retval = (struct sctp_ssnmap *)kzalloc(sizeof(struct sctp_ssnmap), gfp);
 	if (!retval)
 		goto fail;
 
-	if (!sctp_ssnmap_init(retval, in, out))
+	if (!sctp_stream_alloc(&retval->in, in, gfp))
+		goto fail_map;
+
+	if (!sctp_stream_alloc(&retval->out, out, gfp))
 		goto fail_map;
 
 	SCTP_DBG_OBJCNT_INC(ssnmap);
@@ -79,54 +142,29 @@  struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
 	return retval;
 
 fail_map:
-	if (size <= KMALLOC_MAX_SIZE)
-		kfree(retval);
-	else
-		free_pages((unsigned long)retval, get_order(size));
+	sctp_stream_free(&retval->in);
+	sctp_stream_free(&retval->out);
+	kfree(retval);
 fail:
 	return NULL;
 }
 
-
-/* Initialize a block of memory as a ssnmap.  */
-static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
-					    __u16 out)
-{
-	memset(map, 0x00, sctp_ssnmap_size(in, out));
-
-	/* Start 'in' stream just after the map header. */
-	map->in.ssn = (__u16 *)&map[1];
-	map->in.len = in;
-
-	/* Start 'out' stream just after 'in'. */
-	map->out.ssn = &map->in.ssn[in];
-	map->out.len = out;
-
-	return map;
-}
-
 /* Clear out the ssnmap streams.  */
 void sctp_ssnmap_clear(struct sctp_ssnmap *map)
 {
-	size_t size;
-
-	size = (map->in.len + map->out.len) * sizeof(__u16);
-	memset(map->in.ssn, 0x00, size);
+	sctp_stream_clear(&map->in);
+	sctp_stream_clear(&map->out);
 }
 
 /* Dispose of a ssnmap.  */
 void sctp_ssnmap_free(struct sctp_ssnmap *map)
 {
-	int size;
-
 	if (unlikely(!map))
 		return;
 
-	size = sctp_ssnmap_size(map->in.len, map->out.len);
-	if (size <= KMALLOC_MAX_SIZE)
-		kfree(map);
-	else
-		free_pages((unsigned long)map, get_order(size));
+	sctp_stream_free(&map->in);
+	sctp_stream_free(&map->out);
+	kfree(map);
 
 	SCTP_DBG_OBJCNT_DEC(ssnmap);
 }

Comments

Denis V. Lunev March 27, 2018, 6:04 p.m.
we can allocate pages on demand right now,
i.e. when it will be really necessary.

Can be done in the next patch of the set.

Den

On 03/27/2018 08:55 PM, Oleg Babin wrote:
> SCTP protocol allocates TCB on receiving INIT and COOKIE ECHO chunks
> which specify input and output stream count. As the total count of
> those streams can be up to (2^16 - 1) of each type, it is possible
> to occupy a fifth order of memory for only one type of streams. Plus
> allocation can happen in a softirq contex with GFP_ATOMIC flag set
> and we actually do not need the memory to be physically contiguous,
> so for performance reasons we allocate memory on a per-page basis.
>
> https://jira.sw.ru/browse/PSBM-82552
>
> Signed-off-by: Oleg Babin <obabin@virtuozzo.com>
> ---
>  include/net/sctp/structs.h |  44 +++++++++++---
>  net/sctp/ssnmap.c          | 142 ++++++++++++++++++++++++++++-----------------
>  2 files changed, 127 insertions(+), 59 deletions(-)
>
> diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h
> index c2f7064..0a09dbe 100644
> --- a/include/net/sctp/structs.h
> +++ b/include/net/sctp/structs.h
> @@ -396,11 +396,29 @@ typedef struct sctp_sender_hb_info {
>   *
>   *  This is the structure we use to track both our outbound and inbound
>   *  SSN, or Stream Sequence Numbers.
> + *
> + *  As the total count of input or output streams can be up to (2^16 - 1)
> + *  of each type, it is possible to occupy a fifth order of memory for
> + *  only one type of streams. Plus allocation can happen in a softirq
> + *  contex with GFP_ATOMIC flag set and we actually do not need the memory
> + *  to be physically contiguous, so for performance reasons we allocate
> + *  memory on a per-page basis.
>   */
>  
> +#define SCTP_MAX_STREAMS	65535
> +#define SCTP_SSN_PAGE_SHIFT	(PAGE_SHIFT - 1)  /* One SSN takes 2 bytes */
> +#define SCTP_SSN_PAGE_SIZE	(1u << SCTP_SSN_PAGE_SHIFT)
> +#define SCTP_SSN_INDEX_MASK	(SCTP_SSN_PAGE_SIZE - 1)
> +#define SCTP_SSN_MAX_PAGES	((SCTP_MAX_STREAMS + SCTP_SSN_PAGE_SIZE - 1) \
> +				 / SCTP_SSN_PAGE_SIZE)
> +
> +struct sctp_ssn_page {
> +	__u16 ssn[SCTP_SSN_PAGE_SIZE];
> +};
> +
>  struct sctp_stream {
> -	__u16 *ssn;
>  	unsigned int len;
> +	struct sctp_ssn_page *pages[SCTP_SSN_MAX_PAGES];
>  };
>  
>  struct sctp_ssnmap {
> @@ -408,30 +426,42 @@ struct sctp_ssnmap {
>  	struct sctp_stream out;
>  };
>  
> -struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> -				    gfp_t gfp);
> +struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp);
>  void sctp_ssnmap_free(struct sctp_ssnmap *map);
>  void sctp_ssnmap_clear(struct sctp_ssnmap *map);
>  
> +/* Helper function to get a pointer to a particular SSN storage. */
> +static inline __u16 *sctp_ssn_ptr(struct sctp_stream *stream, __u16 id)
> +{
> +	unsigned int page = id >> SCTP_SSN_PAGE_SHIFT;
> +	unsigned int index = id & SCTP_SSN_INDEX_MASK;
> +
> +	return &stream->pages[page]->ssn[index];
> +}
> +
>  /* What is the current SSN number for this stream? */
>  static inline __u16 sctp_ssn_peek(struct sctp_stream *stream, __u16 id)
>  {
> -	return stream->ssn[id];
> +	__u16 *ssnp = sctp_ssn_ptr(stream, id);
> +	return *ssnp;
>  }
>  
>  /* Return the next SSN number for this stream.	*/
>  static inline __u16 sctp_ssn_next(struct sctp_stream *stream, __u16 id)
>  {
> -	return stream->ssn[id]++;
> +	__u16 *ssnp = sctp_ssn_ptr(stream, id);
> +	return (*ssnp)++;
>  }
>  
>  /* Skip over this ssn and all below. */
>  static inline void sctp_ssn_skip(struct sctp_stream *stream, __u16 id, 
>  				 __u16 ssn)
>  {
> -	stream->ssn[id] = ssn+1;
> +	__u16 *ssnp = sctp_ssn_ptr(stream, id);
> +	*ssnp = ssn + 1;
> +
>  }
> -              
> +
>  /*
>   * Pointers to address related SCTP functions.
>   * (i.e. things that depend on the address family.)
> diff --git a/net/sctp/ssnmap.c b/net/sctp/ssnmap.c
> index da86035..610ba6a 100644
> --- a/net/sctp/ssnmap.c
> +++ b/net/sctp/ssnmap.c
> @@ -41,37 +41,100 @@
>  #include <net/sctp/sctp.h>
>  #include <net/sctp/sm.h>
>  
> -static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
> -					    __u16 out);
> +/* Allocate one memory page of SSN storage. */
> +static inline int sctp_ssn_page_alloc(struct sctp_ssn_page **pagep, gfp_t gfp)
> +{
> +	BUILD_BUG_ON(sizeof(struct sctp_ssn_page) != PAGE_SIZE);
>  
> -/* Storage size needed for map includes 2 headers and then the
> - * specific needs of in or out streams.
> - */
> -static inline size_t sctp_ssnmap_size(__u16 in, __u16 out)
> +	*pagep = (struct sctp_ssn_page *)get_zeroed_page(gfp);
> +	return *pagep != NULL;
> +}
> +
> +/* Free one memory page of SSN storage. */
> +static inline void sctp_ssn_page_free(struct sctp_ssn_page *page)
> +{
> +	if (page)
> +		free_pages((unsigned long)page, 0);
> +}
> +
> +/* Allocate memory for SSNs taking less than one memory page. */
> +static inline int sctp_ssn_page_partial_alloc(struct sctp_ssn_page **pagep,
> +					      unsigned int len, gfp_t gfp)
> +{
> +	if (len == 0)
> +		return 1;
> +
> +	*pagep = (struct sctp_ssn_page *)kzalloc(len * sizeof(__u16), gfp);
> +	return *pagep != NULL;
> +}
> +
> +/* Free memory that is smaller than one memory page. */
> +static inline void sctp_ssn_page_partial_free(struct sctp_ssn_page *page)
>  {
> -	return sizeof(struct sctp_ssnmap) + (in + out) * sizeof(__u16);
> +	if (page)
> +		kfree(page);
> +}
> +
> +/* Allocate memory pages for one type of stream (in or out). */
> +static int sctp_stream_alloc(struct sctp_stream *stream, __u16 len, gfp_t gfp)
> +{
> +	unsigned int pages = len >> SCTP_SSN_PAGE_SHIFT;
> +	unsigned int rest = len & SCTP_SSN_INDEX_MASK;
> +	unsigned int pi;
> +
> +	stream->len = len;
> +
> +	for (pi = 0; pi < pages; ++pi)
> +		if (!sctp_ssn_page_alloc(&stream->pages[pi], gfp))
> +			return 0;
> +
> +	if (!sctp_ssn_page_partial_alloc(&stream->pages[pi], rest, gfp))
> +		return 0;
> +
> +	return 1;
>  }
>  
> +/* Free memory pages for one type of stream (in or out). */
> +static void sctp_stream_free(struct sctp_stream *stream)
> +{
> +	unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
> +	unsigned int pi;
> +
> +	for (pi = 0; pi < pages; ++pi)
> +		sctp_ssn_page_free(stream->pages[pi]);
> +
> +	sctp_ssn_page_partial_free(stream->pages[pi]);
> +}
> +
> +/* Clear all SSNs for one type of stream (in or out). */
> +static void sctp_stream_clear(struct sctp_stream *stream)
> +{
> +	unsigned int pages = stream->len >> SCTP_SSN_PAGE_SHIFT;
> +	unsigned int rest = stream->len & SCTP_SSN_INDEX_MASK;
> +	unsigned int pi;
> +
> +	for (pi = 0; pi < pages; ++pi)
> +		memset(stream->pages[pi], 0, sizeof(struct sctp_ssn_page));
> +
> +	if (rest)
> +		memset(stream->pages[pi], 0, rest * sizeof(__u16));
> +}
>  
>  /* Create a new sctp_ssnmap.
> - * Allocate room to store at least 'len' contiguous TSNs.
> + * Allocate room to store at least 'in' + 'out' SSNs.
>   */
> -struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
> -				    gfp_t gfp)
> +struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out, gfp_t gfp)
>  {
>  	struct sctp_ssnmap *retval;
> -	int size;
> -
> -	size = sctp_ssnmap_size(in, out);
> -	if (size <= KMALLOC_MAX_SIZE)
> -		retval = kmalloc(size, gfp);
> -	else
> -		retval = (struct sctp_ssnmap *)
> -			  __get_free_pages(gfp, get_order(size));
> +
> +	retval = (struct sctp_ssnmap *)kzalloc(sizeof(struct sctp_ssnmap), gfp);
>  	if (!retval)
>  		goto fail;
>  
> -	if (!sctp_ssnmap_init(retval, in, out))
> +	if (!sctp_stream_alloc(&retval->in, in, gfp))
> +		goto fail_map;
> +
> +	if (!sctp_stream_alloc(&retval->out, out, gfp))
>  		goto fail_map;
>  
>  	SCTP_DBG_OBJCNT_INC(ssnmap);
> @@ -79,54 +142,29 @@ struct sctp_ssnmap *sctp_ssnmap_new(__u16 in, __u16 out,
>  	return retval;
>  
>  fail_map:
> -	if (size <= KMALLOC_MAX_SIZE)
> -		kfree(retval);
> -	else
> -		free_pages((unsigned long)retval, get_order(size));
> +	sctp_stream_free(&retval->in);
> +	sctp_stream_free(&retval->out);
> +	kfree(retval);
>  fail:
>  	return NULL;
>  }
>  
> -
> -/* Initialize a block of memory as a ssnmap.  */
> -static struct sctp_ssnmap *sctp_ssnmap_init(struct sctp_ssnmap *map, __u16 in,
> -					    __u16 out)
> -{
> -	memset(map, 0x00, sctp_ssnmap_size(in, out));
> -
> -	/* Start 'in' stream just after the map header. */
> -	map->in.ssn = (__u16 *)&map[1];
> -	map->in.len = in;
> -
> -	/* Start 'out' stream just after 'in'. */
> -	map->out.ssn = &map->in.ssn[in];
> -	map->out.len = out;
> -
> -	return map;
> -}
> -
>  /* Clear out the ssnmap streams.  */
>  void sctp_ssnmap_clear(struct sctp_ssnmap *map)
>  {
> -	size_t size;
> -
> -	size = (map->in.len + map->out.len) * sizeof(__u16);
> -	memset(map->in.ssn, 0x00, size);
> +	sctp_stream_clear(&map->in);
> +	sctp_stream_clear(&map->out);
>  }
>  
>  /* Dispose of a ssnmap.  */
>  void sctp_ssnmap_free(struct sctp_ssnmap *map)
>  {
> -	int size;
> -
>  	if (unlikely(!map))
>  		return;
>  
> -	size = sctp_ssnmap_size(map->in.len, map->out.len);
> -	if (size <= KMALLOC_MAX_SIZE)
> -		kfree(map);
> -	else
> -		free_pages((unsigned long)map, get_order(size));
> +	sctp_stream_free(&map->in);
> +	sctp_stream_free(&map->out);
> +	kfree(map);
>  
>  	SCTP_DBG_OBJCNT_DEC(ssnmap);
>  }