[RHEL8,COMMIT] mm/memcg: Use per-cpu stock charges for ->kmem and ->cache counters #PSBM-101300

Submitted by Konstantin Khorenko on Oct. 14, 2020, 12:45 p.m.

Details

Message ID 202010141245.09ECjpfd1528808@finist-co8.sw.ru
State New
Series "mm/memcg: Use per-cpu stock charges for ->kmem and ->cache counters"
Headers show

Commit Message

Konstantin Khorenko Oct. 14, 2020, 12:45 p.m.
The commit is pushed to "branch-rh8-4.18.0-193.6.3.vz8.4.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh8-4.18.0-193.6.3.vz8.4.12
------>
commit 84476bf2ce81c2fb9454adc048fbc9e9a0704538
Author: Andrey Ryabinin <aryabinin@virtuozzo.com>
Date:   Wed Oct 14 15:45:51 2020 +0300

    mm/memcg: Use per-cpu stock charges for ->kmem and ->cache counters #PSBM-101300
    
    Currently we use per-cpu stocks to do precharges of the ->memory and ->memsw
    counters. Do this for the ->kmem and ->cache as well to decrease contention
    on these counters as well.
    
    https://jira.sw.ru/browse/PSBM-101300
    Signed-off-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
---
 mm/memcontrol.c | 75 +++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 51 insertions(+), 24 deletions(-)

Patch hide | download patch | download mbox

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 134cb27307f2..b3f97309ca39 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2023,6 +2023,8 @@  EXPORT_SYMBOL(unlock_page_memcg);
 struct memcg_stock_pcp {
 	struct mem_cgroup *cached; /* this never be root cgroup */
 	unsigned int nr_pages;
+	unsigned int cache_nr_pages;
+	unsigned int kmem_nr_pages;
 	struct work_struct work;
 	unsigned long flags;
 #define FLUSHING_CACHED_CHARGE	0
@@ -2041,7 +2043,8 @@  static DEFINE_MUTEX(percpu_charge_mutex);
  *
  * returns true if successful, false otherwise.
  */
-static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
+static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages,
+			bool cache, bool kmem)
 {
 	struct memcg_stock_pcp *stock;
 	unsigned long flags;
@@ -2053,9 +2056,19 @@  static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 	local_irq_save(flags);
 
 	stock = this_cpu_ptr(&memcg_stock);
-	if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
-		stock->nr_pages -= nr_pages;
-		ret = true;
+	if (memcg == stock->cached) {
+		if (cache && stock->cache_nr_pages >= nr_pages) {
+			stock->cache_nr_pages -= nr_pages;
+			ret = true;
+		}
+		if (kmem && stock->kmem_nr_pages >= nr_pages) {
+			stock->kmem_nr_pages -= nr_pages;
+			ret = true;
+		}
+		if (!cache && !kmem && stock->nr_pages >= nr_pages) {
+			stock->nr_pages -= nr_pages;
+			ret = true;
+		}
 	}
 
 	local_irq_restore(flags);
@@ -2069,13 +2082,21 @@  static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 static void drain_stock(struct memcg_stock_pcp *stock)
 {
 	struct mem_cgroup *old = stock->cached;
+	unsigned long nr_pages = stock->nr_pages + stock->cache_nr_pages + stock->kmem_nr_pages;
+
+	if (stock->cache_nr_pages)
+		page_counter_uncharge(&old->cache, stock->cache_nr_pages);
+	if (stock->kmem_nr_pages)
+		page_counter_uncharge(&old->kmem, stock->kmem_nr_pages);
 
-	if (stock->nr_pages) {
-		page_counter_uncharge(&old->memory, stock->nr_pages);
+	if (nr_pages) {
+		page_counter_uncharge(&old->memory, nr_pages);
 		if (do_memsw_account())
-			page_counter_uncharge(&old->memsw, stock->nr_pages);
+			page_counter_uncharge(&old->memsw, nr_pages);
 		css_put_many(&old->css, stock->nr_pages);
 		stock->nr_pages = 0;
+		stock->kmem_nr_pages = 0;
+		stock->cache_nr_pages = 0;
 	}
 	stock->cached = NULL;
 }
@@ -2102,10 +2123,12 @@  static void drain_local_stock(struct work_struct *dummy)
  * Cache charges(val) to local per_cpu area.
  * This will be consumed by consume_stock() function, later.
  */
-static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
+static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages,
+			bool cache, bool kmem)
 {
 	struct memcg_stock_pcp *stock;
 	unsigned long flags;
+	unsigned long stock_nr_pages;
 
 	local_irq_save(flags);
 
@@ -2114,9 +2137,17 @@  static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 		drain_stock(stock);
 		stock->cached = memcg;
 	}
-	stock->nr_pages += nr_pages;
 
-	if (stock->nr_pages > MEMCG_CHARGE_BATCH)
+	if (cache)
+		stock->cache_nr_pages += nr_pages;
+	else if (kmem)
+		stock->kmem_nr_pages += nr_pages;
+	else
+		stock->nr_pages += nr_pages;
+
+	stock_nr_pages = stock->nr_pages + stock->cache_nr_pages +
+		stock->kmem_nr_pages;
+	if (nr_pages > MEMCG_CHARGE_BATCH)
 		drain_stock(stock);
 
 	local_irq_restore(flags);
@@ -2143,9 +2174,11 @@  static void drain_all_stock(struct mem_cgroup *root_memcg)
 	for_each_online_cpu(cpu) {
 		struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
 		struct mem_cgroup *memcg;
+		unsigned long nr_pages = stock->nr_pages + stock->kmem_nr_pages +
+			stock->cache_nr_pages;
 
 		memcg = stock->cached;
-		if (!memcg || !stock->nr_pages || !css_tryget(&memcg->css))
+		if (!memcg || !nr_pages || !css_tryget(&memcg->css))
 			continue;
 		if (!mem_cgroup_is_descendant(memcg, root_memcg)) {
 			css_put(&memcg->css);
@@ -2273,17 +2306,11 @@  static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge
 retry:
 	may_swap = true;
 	kmem_limit = false;
-	if (consume_stock(memcg, nr_pages)) {
-		if (kmem_charge && !page_counter_try_charge(
-				&memcg->kmem, nr_pages, &counter)) {
-			refill_stock(memcg, nr_pages);
-			goto charge;
-		}
+	if (consume_stock(memcg, nr_pages, cache_charge, kmem_charge)) {
 		css_get_many(&memcg->css, batch);
 		goto done;
 	}
 
-charge:
 	mem_over_limit = NULL;
 	if (page_counter_try_charge(&memcg->memory, batch, &counter)) {
 		if (do_memsw_account() && !page_counter_try_charge(
@@ -2296,7 +2323,7 @@  static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge
 		mem_over_limit = mem_cgroup_from_counter(counter, memory);
 
 	if (!mem_over_limit && kmem_charge) {
-		if (!page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
+		if (!page_counter_try_charge(&memcg->kmem, batch, &counter)) {
 			kmem_limit = true;
 			mem_over_limit = mem_cgroup_from_counter(counter, kmem);
 			page_counter_uncharge(&memcg->memory, batch);
@@ -2425,13 +2452,13 @@  static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, bool kmem_charge
 	return 0;
 
 done_restock:
+	if (cache_charge)
+		page_counter_charge(&memcg->cache, batch);
+
 	css_get_many(&memcg->css, batch);
 	if (batch > nr_pages)
-		refill_stock(memcg, batch - nr_pages);
+		refill_stock(memcg, batch - nr_pages, cache_charge, kmem_charge);
 done:
-	if (cache_charge)
-		page_counter_charge(&memcg->cache, nr_pages);
-
 	/*
 	 * If the hierarchy is above the normal consumption range, schedule
 	 * reclaim on returning to userland.  We can perform reclaim here
@@ -6972,7 +6999,7 @@  void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
 
 	mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
 
-	refill_stock(memcg, nr_pages);
+	refill_stock(memcg, nr_pages, false, false);
 }
 
 static int __init cgroup_memory(char *s)