target/iscsi: add an ability to set io limits for iscsi targets

Submitted by Andrei Vagin on March 13, 2018, 10:48 p.m.

Details

Message ID 1520981282-5605-1-git-send-email-avagin@openvz.org
State New
Series "target/iscsi: add an ability to set io limits for iscsi targets"
Headers show

Commit Message

Andrei Vagin March 13, 2018, 10:48 p.m.
This patch add an ability to set a blkio cgroup for an iscsi target.

When a new client is connected, the kernel creates two kernel threads and
run all io requests from them, this means that we can set a blkio group
for these threads and set io limits via this group.

Here is an exampe how this works:

  $ TGT_PATH=/sys/kernel/config/target/iscsi/iqn.2014-06.com.vstorage\:test
  $ CG_PATH=/sys/fs/cgroup/blkio/system.slice/vstorage-iscsi/

  # create a group
  $ mkdir -p $CG_PATH

  # set cgroup for iscsi target
  $ bash -c 'echo $$ > $CG_PATH/tasks &&
             echo 1 > $TGT_PATH/tpgt_1/param/BlkioCgroup'
  $ cat $TGT_PATH/tpgt_1/param/BlkioCgroup
  /system.slice/vstorage-iscsi

  # attach iscsi target
  $ IQN=iqn.2014-06.com.vstorage:test
  $ iscsiadm -m node -T $IQN -l
  Logging in to [iface: default, target: iqn.2014-06.com.vstorage:test,
  portal: 10.94.120.187,3260] (multiple)
  Login to [iface: default, target: iqn.2014-06.com.vstorage:test, portal:
  10.94.120.187,3260] successful.

  # check that iscsi threads in the required cgroup
  $ ps -C iscsi_ttx
   PID TTY          TIME CMD
   4097 ?        00:00:00 iscsi_ttx
  $ cat /proc/4097/cgroup | grep blkio
  1:blkio:/system.slice/vstorage-iscsi

  # set io limits for a target backing store device
  $ ploop list
  ploop15810 /mnt/vstorage/vols/iscsi/iqn.2014-06.com.vstorage:test/lun1/ploop
  $ ls -l /dev/ploop15810
  brw-rw---- 1 root disk 182, 252960 Mar 14 00:59 /dev/ploop15810
  $ echo "182:252960 6291456" > $CG_PATH/blkio.throttle.read_bps_device

  # check that limits work as expected
  $ dd if=/dev/sda of=/dev/null bs=10M count=10
  10+0 records in
  10+0 records out
  104857600 bytes (105 MB) copied, 17.3534 s, 6.0 MB/s

Signed-off-by: Andrei Vagin <avagin@openvz.org>
---
 drivers/target/iscsi/iscsi_target_configfs.c | 46 ++++++++++++++++++++++++++++
 drivers/target/iscsi/iscsi_target_login.c    | 34 +++++++++++++++++++-
 drivers/target/iscsi/iscsi_target_tpg.c      | 36 ++++++++++++++++++++++
 drivers/target/iscsi/iscsi_target_tpg.h      |  2 ++
 include/target/iscsi/iscsi_target_core.h     |  2 ++
 kernel/cgroup.c                              |  1 +
 6 files changed, 120 insertions(+), 1 deletion(-)

Patch hide | download patch | download mbox

diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
index e6631ab..6d754cf 100644
--- a/drivers/target/iscsi/iscsi_target_configfs.c
+++ b/drivers/target/iscsi/iscsi_target_configfs.c
@@ -1412,6 +1412,51 @@  TPG_PARAM_ATTR(IFMarkInt, S_IRUGO | S_IWUSR);
 DEF_TPG_PARAM(OFMarkInt);
 TPG_PARAM_ATTR(OFMarkInt, S_IRUGO | S_IWUSR);
 
+static ssize_t iscsi_tpg_param_show_BlkioCgroup(
+	struct se_portal_group *se_tpg,
+	char *page)
+{
+	struct iscsi_portal_group *tpg = container_of(se_tpg,
+			struct iscsi_portal_group, tpg_se_tpg);
+	ssize_t rb;
+
+	if (iscsit_get_tpg(tpg) < 0)
+		return -EINVAL;
+
+	rb = iscsit_ta_tpg_show_blkcg(tpg, page);
+	iscsit_put_tpg(tpg);
+	return rb;
+}
+
+static ssize_t iscsi_tpg_param_store_BlkioCgroup(
+	struct se_portal_group *se_tpg,
+	const char *page,
+	size_t count)
+{
+	struct iscsi_portal_group *tpg = container_of(se_tpg,
+			struct iscsi_portal_group, tpg_se_tpg);
+	u32 val;
+	int ret;
+
+	if (iscsit_get_tpg(tpg) < 0)
+		return -EINVAL;
+
+	ret = kstrtou32(page, 0, &val);
+	if (ret)
+		goto out;
+	ret = iscsit_ta_tpg_set_blkcg(tpg, val);
+	if (ret < 0)
+		goto out;
+
+	iscsit_put_tpg(tpg);
+	return count;
+out:
+	iscsit_put_tpg(tpg);
+	return ret;
+}
+
+TPG_PARAM_ATTR(BlkioCgroup, S_IRUGO | S_IWUSR);
+
 static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
 	&iscsi_tpg_param_AuthMethod.attr,
 	&iscsi_tpg_param_HeaderDigest.attr,
@@ -1434,6 +1479,7 @@  static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
 	&iscsi_tpg_param_OFMarker.attr,
 	&iscsi_tpg_param_IFMarkInt.attr,
 	&iscsi_tpg_param_OFMarkInt.attr,
+	&iscsi_tpg_param_BlkioCgroup.attr,
 	NULL,
 };
 
diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
index c20b561..4f59416 100644
--- a/drivers/target/iscsi/iscsi_target_login.c
+++ b/drivers/target/iscsi/iscsi_target_login.c
@@ -20,6 +20,7 @@ 
 #include <linux/string.h>
 #include <linux/kthread.h>
 #include <linux/idr.h>
+#include <linux/cgroup.h>
 #include <scsi/iscsi_proto.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
@@ -712,8 +713,18 @@  static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
 
 int iscsit_start_kthreads(struct iscsi_conn *conn)
 {
+	struct iscsi_portal_group *tpg = conn->tpg;
+	struct cgroup_subsys_state *blk_css = NULL;
 	int ret = 0;
 
+	if (iscsit_get_tpg(tpg) < 0)
+		return -EINVAL;
+	if (tpg->blk_css) {
+		blk_css = tpg->blk_css;
+		css_get(blk_css);
+	}
+	iscsit_put_tpg(tpg);
+
 	spin_lock(&iscsit_global->ts_bitmap_lock);
 	conn->bitmap_id = bitmap_find_free_region(iscsit_global->ts_bitmap,
 					ISCSIT_BITMAP_BITS, get_order(1));
@@ -722,7 +733,8 @@  int iscsit_start_kthreads(struct iscsi_conn *conn)
 	if (conn->bitmap_id < 0) {
 		pr_err("bitmap_find_free_region() failed for"
 		       " iscsit_start_kthreads()\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto put_blk_css;
 	}
 
 	conn->tx_thread = kthread_run(iscsi_target_tx_thread, conn,
@@ -732,6 +744,11 @@  int iscsit_start_kthreads(struct iscsi_conn *conn)
 		ret = PTR_ERR(conn->tx_thread);
 		goto out_bitmap;
 	}
+	if (blk_css) {
+		ret = cgroup_kernel_attach(blk_css->cgroup, conn->tx_thread);
+		if (ret < 0)
+			goto out_tx;
+	}
 	conn->tx_thread_active = true;
 
 	conn->rx_thread = kthread_run(iscsi_target_rx_thread, conn,
@@ -741,9 +758,21 @@  int iscsit_start_kthreads(struct iscsi_conn *conn)
 		ret = PTR_ERR(conn->rx_thread);
 		goto out_tx;
 	}
+	if (blk_css) {
+		ret = cgroup_kernel_attach(blk_css->cgroup, conn->rx_thread);
+		if (ret < 0)
+			goto out_rx;
+	}
 	conn->rx_thread_active = true;
 
+	if (blk_css)
+		css_put(blk_css);
+
 	return 0;
+out_rx:
+	send_sig(SIGINT, conn->rx_thread, 1);
+	kthread_stop(conn->rx_thread);
+	conn->rx_thread_active = false;
 out_tx:
 	send_sig(SIGINT, conn->tx_thread, 1);
 	kthread_stop(conn->tx_thread);
@@ -753,6 +782,9 @@  out_bitmap:
 	bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
 			      get_order(1));
 	spin_unlock(&iscsit_global->ts_bitmap_lock);
+put_blk_css:
+	if (blk_css)
+		css_put(blk_css);
 	return ret;
 }
 
diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
index 2426b0b..792d3ee 100644
--- a/drivers/target/iscsi/iscsi_target_tpg.c
+++ b/drivers/target/iscsi/iscsi_target_tpg.c
@@ -16,6 +16,7 @@ 
  * GNU General Public License for more details.
  ******************************************************************************/
 
+#include <linux/cgroup.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
 #include <target/target_core_configfs.h>
@@ -303,6 +304,9 @@  int iscsit_tpg_del_portal_group(
 	list_del(&tpg->tpg_list);
 	spin_unlock(&tiqn->tiqn_tpg_lock);
 
+	if (tpg->blk_css)
+		css_put(tpg->blk_css);
+
 	pr_debug("CORE[%s]_TPG[%hu] - Deleted iSCSI Target Portal Group\n",
 			tiqn->tiqn, tpg->tpgt);
 
@@ -896,3 +900,35 @@  int iscsit_ta_tpg_enabled_sendtargets(
 
 	return 0;
 }
+
+int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag)
+{
+	struct cgroup_subsys_state *css;
+
+	if (flag != 1)
+		return -EINVAL;
+
+	css = task_get_css(current, blkio_subsys_id);
+	if (tpg->blk_css)
+		css_put(tpg->blk_css);
+	tpg->blk_css = css;
+
+	return 0;
+}
+
+int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page)
+{
+	int rb;
+
+	if (tpg->blk_css) {
+		rb = cgroup_path(tpg->blk_css->cgroup, page, PAGE_SIZE - 1);
+		if (rb == 0)
+			rb = strlen(page);
+		page[rb] = '\n';
+		page[rb + 1] = 0;
+		rb++;
+	} else
+		rb = 0;
+
+	return rb;
+}
diff --git a/drivers/target/iscsi/iscsi_target_tpg.h b/drivers/target/iscsi/iscsi_target_tpg.h
index b2f49c0..416b929 100644
--- a/drivers/target/iscsi/iscsi_target_tpg.h
+++ b/drivers/target/iscsi/iscsi_target_tpg.h
@@ -40,5 +40,7 @@  extern int iscsit_ta_demo_mode_discovery(struct iscsi_portal_group *, u32);
 extern int iscsit_ta_default_erl(struct iscsi_portal_group *, u32);
 extern int iscsit_ta_t10_pi(struct iscsi_portal_group *, u32);
 extern int iscsit_ta_tpg_enabled_sendtargets(struct iscsi_portal_group *, u32);
+extern int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag);
+extern int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page);
 
 #endif /* ISCSI_TARGET_TPG_H */
diff --git a/include/target/iscsi/iscsi_target_core.h b/include/target/iscsi/iscsi_target_core.h
index 3e3b2f9..83eb63b 100644
--- a/include/target/iscsi/iscsi_target_core.h
+++ b/include/target/iscsi/iscsi_target_core.h
@@ -838,6 +838,8 @@  struct iscsi_portal_group {
 	struct iscsi_tiqn	*tpg_tiqn;
 	struct list_head	tpg_gnp_list;
 	struct list_head	tpg_list;
+
+	struct cgroup_subsys_state *blk_css;
 } ____cacheline_aligned;
 
 struct iscsi_wwn_stat_grps {
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 11c547f..0fbb37e 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -5924,6 +5924,7 @@  int cgroup_kernel_attach(struct cgroup *cgrp, struct task_struct *tsk)
 	mutex_unlock(&cgroup_mutex);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(cgroup_kernel_attach);
 
 void cgroup_kernel_close(struct cgroup *cgrp)
 {

Comments

Vasily Averin March 14, 2018, 8:01 a.m.
See small comment below

On 2018-03-14 01:48, Andrei Vagin wrote:
> This patch add an ability to set a blkio cgroup for an iscsi target.
> 
> When a new client is connected, the kernel creates two kernel threads and
> run all io requests from them, this means that we can set a blkio group
> for these threads and set io limits via this group.
> 
> Here is an exampe how this works:
> 
>   $ TGT_PATH=/sys/kernel/config/target/iscsi/iqn.2014-06.com.vstorage\:test
>   $ CG_PATH=/sys/fs/cgroup/blkio/system.slice/vstorage-iscsi/
> 
>   # create a group
>   $ mkdir -p $CG_PATH
> 
>   # set cgroup for iscsi target
>   $ bash -c 'echo $$ > $CG_PATH/tasks &&
>              echo 1 > $TGT_PATH/tpgt_1/param/BlkioCgroup'
>   $ cat $TGT_PATH/tpgt_1/param/BlkioCgroup
>   /system.slice/vstorage-iscsi
> 
>   # attach iscsi target
>   $ IQN=iqn.2014-06.com.vstorage:test
>   $ iscsiadm -m node -T $IQN -l
>   Logging in to [iface: default, target: iqn.2014-06.com.vstorage:test,
>   portal: 10.94.120.187,3260] (multiple)
>   Login to [iface: default, target: iqn.2014-06.com.vstorage:test, portal:
>   10.94.120.187,3260] successful.
> 
>   # check that iscsi threads in the required cgroup
>   $ ps -C iscsi_ttx
>    PID TTY          TIME CMD
>    4097 ?        00:00:00 iscsi_ttx
>   $ cat /proc/4097/cgroup | grep blkio
>   1:blkio:/system.slice/vstorage-iscsi
> 
>   # set io limits for a target backing store device
>   $ ploop list
>   ploop15810 /mnt/vstorage/vols/iscsi/iqn.2014-06.com.vstorage:test/lun1/ploop
>   $ ls -l /dev/ploop15810
>   brw-rw---- 1 root disk 182, 252960 Mar 14 00:59 /dev/ploop15810
>   $ echo "182:252960 6291456" > $CG_PATH/blkio.throttle.read_bps_device
> 
>   # check that limits work as expected
>   $ dd if=/dev/sda of=/dev/null bs=10M count=10
>   10+0 records in
>   10+0 records out
>   104857600 bytes (105 MB) copied, 17.3534 s, 6.0 MB/s
> 
> Signed-off-by: Andrei Vagin <avagin@openvz.org>
> ---
>  drivers/target/iscsi/iscsi_target_configfs.c | 46 ++++++++++++++++++++++++++++
>  drivers/target/iscsi/iscsi_target_login.c    | 34 +++++++++++++++++++-
>  drivers/target/iscsi/iscsi_target_tpg.c      | 36 ++++++++++++++++++++++
>  drivers/target/iscsi/iscsi_target_tpg.h      |  2 ++
>  include/target/iscsi/iscsi_target_core.h     |  2 ++
>  kernel/cgroup.c                              |  1 +
>  6 files changed, 120 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
> index e6631ab..6d754cf 100644
> --- a/drivers/target/iscsi/iscsi_target_configfs.c
> +++ b/drivers/target/iscsi/iscsi_target_configfs.c
> @@ -1412,6 +1412,51 @@ TPG_PARAM_ATTR(IFMarkInt, S_IRUGO | S_IWUSR);
>  DEF_TPG_PARAM(OFMarkInt);
>  TPG_PARAM_ATTR(OFMarkInt, S_IRUGO | S_IWUSR);
>  
> +static ssize_t iscsi_tpg_param_show_BlkioCgroup(
> +	struct se_portal_group *se_tpg,
> +	char *page)
> +{
> +	struct iscsi_portal_group *tpg = container_of(se_tpg,
> +			struct iscsi_portal_group, tpg_se_tpg);
> +	ssize_t rb;
> +
> +	if (iscsit_get_tpg(tpg) < 0)
> +		return -EINVAL;
> +
> +	rb = iscsit_ta_tpg_show_blkcg(tpg, page);
> +	iscsit_put_tpg(tpg);
> +	return rb;
> +}
> +
> +static ssize_t iscsi_tpg_param_store_BlkioCgroup(
> +	struct se_portal_group *se_tpg,
> +	const char *page,
> +	size_t count)
> +{
> +	struct iscsi_portal_group *tpg = container_of(se_tpg,
> +			struct iscsi_portal_group, tpg_se_tpg);
> +	u32 val;
> +	int ret;
> +
> +	if (iscsit_get_tpg(tpg) < 0)
> +		return -EINVAL;
> +
> +	ret = kstrtou32(page, 0, &val);
> +	if (ret)
> +		goto out;
> +	ret = iscsit_ta_tpg_set_blkcg(tpg, val);
> +	if (ret < 0)
> +		goto out;
> +
> +	iscsit_put_tpg(tpg);
> +	return count;
> +out:
> +	iscsit_put_tpg(tpg);
> +	return ret;
> +}
> +
> +TPG_PARAM_ATTR(BlkioCgroup, S_IRUGO | S_IWUSR);
> +
>  static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
>  	&iscsi_tpg_param_AuthMethod.attr,
>  	&iscsi_tpg_param_HeaderDigest.attr,
> @@ -1434,6 +1479,7 @@ static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
>  	&iscsi_tpg_param_OFMarker.attr,
>  	&iscsi_tpg_param_IFMarkInt.attr,
>  	&iscsi_tpg_param_OFMarkInt.attr,
> +	&iscsi_tpg_param_BlkioCgroup.attr,
>  	NULL,
>  };
>  
> diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
> index c20b561..4f59416 100644
> --- a/drivers/target/iscsi/iscsi_target_login.c
> +++ b/drivers/target/iscsi/iscsi_target_login.c
> @@ -20,6 +20,7 @@
>  #include <linux/string.h>
>  #include <linux/kthread.h>
>  #include <linux/idr.h>
> +#include <linux/cgroup.h>
>  #include <scsi/iscsi_proto.h>
>  #include <target/target_core_base.h>
>  #include <target/target_core_fabric.h>
> @@ -712,8 +713,18 @@ static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
>  
>  int iscsit_start_kthreads(struct iscsi_conn *conn)
>  {
> +	struct iscsi_portal_group *tpg = conn->tpg;
> +	struct cgroup_subsys_state *blk_css = NULL;
>  	int ret = 0;
>  
> +	if (iscsit_get_tpg(tpg) < 0)
> +		return -EINVAL;
> +	if (tpg->blk_css) {
> +		blk_css = tpg->blk_css;
> +		css_get(blk_css);
> +	}
> +	iscsit_put_tpg(tpg);
> +
>  	spin_lock(&iscsit_global->ts_bitmap_lock);
>  	conn->bitmap_id = bitmap_find_free_region(iscsit_global->ts_bitmap,
>  					ISCSIT_BITMAP_BITS, get_order(1));
> @@ -722,7 +733,8 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
>  	if (conn->bitmap_id < 0) {
>  		pr_err("bitmap_find_free_region() failed for"
>  		       " iscsit_start_kthreads()\n");
> -		return -ENOMEM;
> +		ret = -ENOMEM;
> +		goto put_blk_css;
>  	}
>  
>  	conn->tx_thread = kthread_run(iscsi_target_tx_thread, conn,
> @@ -732,6 +744,11 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
>  		ret = PTR_ERR(conn->tx_thread);
>  		goto out_bitmap;
>  	}
> +	if (blk_css) {
> +		ret = cgroup_kernel_attach(blk_css->cgroup, conn->tx_thread);
> +		if (ret < 0)
> +			goto out_tx;
> +	}
>  	conn->tx_thread_active = true;
>  
>  	conn->rx_thread = kthread_run(iscsi_target_rx_thread, conn,
> @@ -741,9 +758,21 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
>  		ret = PTR_ERR(conn->rx_thread);
>  		goto out_tx;
>  	}
> +	if (blk_css) {
> +		ret = cgroup_kernel_attach(blk_css->cgroup, conn->rx_thread);
> +		if (ret < 0)
> +			goto out_rx;
> +	}
>  	conn->rx_thread_active = true;
>  
> +	if (blk_css)
> +		css_put(blk_css);
> +
>  	return 0;
> +out_rx:
> +	send_sig(SIGINT, conn->rx_thread, 1);
> +	kthread_stop(conn->rx_thread);
> +	conn->rx_thread_active = false;

Seems rx_thread_active = false can be skipped here, 
it is set to true in success case only,
after goto to this label.

>  out_tx:
>  	send_sig(SIGINT, conn->tx_thread, 1);
>  	kthread_stop(conn->tx_thread);
> @@ -753,6 +782,9 @@ out_bitmap:
>  	bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
>  			      get_order(1));
>  	spin_unlock(&iscsit_global->ts_bitmap_lock);
> +put_blk_css:
> +	if (blk_css)
> +		css_put(blk_css);
>  	return ret;
>  }
>  
> diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
> index 2426b0b..792d3ee 100644
> --- a/drivers/target/iscsi/iscsi_target_tpg.c
> +++ b/drivers/target/iscsi/iscsi_target_tpg.c
> @@ -16,6 +16,7 @@
>   * GNU General Public License for more details.
>   ******************************************************************************/
>  
> +#include <linux/cgroup.h>
>  #include <target/target_core_base.h>
>  #include <target/target_core_fabric.h>
>  #include <target/target_core_configfs.h>
> @@ -303,6 +304,9 @@ int iscsit_tpg_del_portal_group(
>  	list_del(&tpg->tpg_list);
>  	spin_unlock(&tiqn->tiqn_tpg_lock);
>  
> +	if (tpg->blk_css)
> +		css_put(tpg->blk_css);
> +
>  	pr_debug("CORE[%s]_TPG[%hu] - Deleted iSCSI Target Portal Group\n",
>  			tiqn->tiqn, tpg->tpgt);
>  
> @@ -896,3 +900,35 @@ int iscsit_ta_tpg_enabled_sendtargets(
>  
>  	return 0;
>  }
> +
> +int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag)
> +{
> +	struct cgroup_subsys_state *css;
> +
> +	if (flag != 1)
> +		return -EINVAL;
> +
> +	css = task_get_css(current, blkio_subsys_id);
> +	if (tpg->blk_css)
> +		css_put(tpg->blk_css);
> +	tpg->blk_css = css;
> +
> +	return 0;
> +}
> +
> +int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page)
> +{
> +	int rb;
> +
> +	if (tpg->blk_css) {
> +		rb = cgroup_path(tpg->blk_css->cgroup, page, PAGE_SIZE - 1);
> +		if (rb == 0)
> +			rb = strlen(page);
> +		page[rb] = '\n';
> +		page[rb + 1] = 0;
> +		rb++;
> +	} else
> +		rb = 0;
> +
> +	return rb;
> +}
> diff --git a/drivers/target/iscsi/iscsi_target_tpg.h b/drivers/target/iscsi/iscsi_target_tpg.h
> index b2f49c0..416b929 100644
> --- a/drivers/target/iscsi/iscsi_target_tpg.h
> +++ b/drivers/target/iscsi/iscsi_target_tpg.h
> @@ -40,5 +40,7 @@ extern int iscsit_ta_demo_mode_discovery(struct iscsi_portal_group *, u32);
>  extern int iscsit_ta_default_erl(struct iscsi_portal_group *, u32);
>  extern int iscsit_ta_t10_pi(struct iscsi_portal_group *, u32);
>  extern int iscsit_ta_tpg_enabled_sendtargets(struct iscsi_portal_group *, u32);
> +extern int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag);
> +extern int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page);
>  
>  #endif /* ISCSI_TARGET_TPG_H */
> diff --git a/include/target/iscsi/iscsi_target_core.h b/include/target/iscsi/iscsi_target_core.h
> index 3e3b2f9..83eb63b 100644
> --- a/include/target/iscsi/iscsi_target_core.h
> +++ b/include/target/iscsi/iscsi_target_core.h
> @@ -838,6 +838,8 @@ struct iscsi_portal_group {
>  	struct iscsi_tiqn	*tpg_tiqn;
>  	struct list_head	tpg_gnp_list;
>  	struct list_head	tpg_list;
> +
> +	struct cgroup_subsys_state *blk_css;
>  } ____cacheline_aligned;
>  
>  struct iscsi_wwn_stat_grps {
> diff --git a/kernel/cgroup.c b/kernel/cgroup.c
> index 11c547f..0fbb37e 100644
> --- a/kernel/cgroup.c
> +++ b/kernel/cgroup.c
> @@ -5924,6 +5924,7 @@ int cgroup_kernel_attach(struct cgroup *cgrp, struct task_struct *tsk)
>  	mutex_unlock(&cgroup_mutex);
>  	return ret;
>  }
> +EXPORT_SYMBOL_GPL(cgroup_kernel_attach);
>  
>  void cgroup_kernel_close(struct cgroup *cgrp)
>  {
>
Andrey Vagin March 21, 2018, 8:49 p.m.
On Wed, Mar 14, 2018 at 11:01:43AM +0300, Vasily Averin wrote:
> See small comment below
> 
> On 2018-03-14 01:48, Andrei Vagin wrote:
> > This patch add an ability to set a blkio cgroup for an iscsi target.
> > 
> > When a new client is connected, the kernel creates two kernel threads and
> > run all io requests from them, this means that we can set a blkio group
> > for these threads and set io limits via this group.
> > 
> > Here is an exampe how this works:
> > 
> >   $ TGT_PATH=/sys/kernel/config/target/iscsi/iqn.2014-06.com.vstorage\:test
> >   $ CG_PATH=/sys/fs/cgroup/blkio/system.slice/vstorage-iscsi/
> > 
> >   # create a group
> >   $ mkdir -p $CG_PATH
> > 
> >   # set cgroup for iscsi target
> >   $ bash -c 'echo $$ > $CG_PATH/tasks &&
> >              echo 1 > $TGT_PATH/tpgt_1/param/BlkioCgroup'
> >   $ cat $TGT_PATH/tpgt_1/param/BlkioCgroup
> >   /system.slice/vstorage-iscsi
> > 
> >   # attach iscsi target
> >   $ IQN=iqn.2014-06.com.vstorage:test
> >   $ iscsiadm -m node -T $IQN -l
> >   Logging in to [iface: default, target: iqn.2014-06.com.vstorage:test,
> >   portal: 10.94.120.187,3260] (multiple)
> >   Login to [iface: default, target: iqn.2014-06.com.vstorage:test, portal:
> >   10.94.120.187,3260] successful.
> > 
> >   # check that iscsi threads in the required cgroup
> >   $ ps -C iscsi_ttx
> >    PID TTY          TIME CMD
> >    4097 ?        00:00:00 iscsi_ttx
> >   $ cat /proc/4097/cgroup | grep blkio
> >   1:blkio:/system.slice/vstorage-iscsi
> > 
> >   # set io limits for a target backing store device
> >   $ ploop list
> >   ploop15810 /mnt/vstorage/vols/iscsi/iqn.2014-06.com.vstorage:test/lun1/ploop
> >   $ ls -l /dev/ploop15810
> >   brw-rw---- 1 root disk 182, 252960 Mar 14 00:59 /dev/ploop15810
> >   $ echo "182:252960 6291456" > $CG_PATH/blkio.throttle.read_bps_device
> > 
> >   # check that limits work as expected
> >   $ dd if=/dev/sda of=/dev/null bs=10M count=10
> >   10+0 records in
> >   10+0 records out
> >   104857600 bytes (105 MB) copied, 17.3534 s, 6.0 MB/s
> > 
> > Signed-off-by: Andrei Vagin <avagin@openvz.org>
> > ---
> >  drivers/target/iscsi/iscsi_target_configfs.c | 46 ++++++++++++++++++++++++++++
> >  drivers/target/iscsi/iscsi_target_login.c    | 34 +++++++++++++++++++-
> >  drivers/target/iscsi/iscsi_target_tpg.c      | 36 ++++++++++++++++++++++
> >  drivers/target/iscsi/iscsi_target_tpg.h      |  2 ++
> >  include/target/iscsi/iscsi_target_core.h     |  2 ++
> >  kernel/cgroup.c                              |  1 +
> >  6 files changed, 120 insertions(+), 1 deletion(-)
> > 
> > diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c
> > index e6631ab..6d754cf 100644
> > --- a/drivers/target/iscsi/iscsi_target_configfs.c
> > +++ b/drivers/target/iscsi/iscsi_target_configfs.c
> > @@ -1412,6 +1412,51 @@ TPG_PARAM_ATTR(IFMarkInt, S_IRUGO | S_IWUSR);
> >  DEF_TPG_PARAM(OFMarkInt);
> >  TPG_PARAM_ATTR(OFMarkInt, S_IRUGO | S_IWUSR);
> >  
> > +static ssize_t iscsi_tpg_param_show_BlkioCgroup(
> > +	struct se_portal_group *se_tpg,
> > +	char *page)
> > +{
> > +	struct iscsi_portal_group *tpg = container_of(se_tpg,
> > +			struct iscsi_portal_group, tpg_se_tpg);
> > +	ssize_t rb;
> > +
> > +	if (iscsit_get_tpg(tpg) < 0)
> > +		return -EINVAL;
> > +
> > +	rb = iscsit_ta_tpg_show_blkcg(tpg, page);
> > +	iscsit_put_tpg(tpg);
> > +	return rb;
> > +}
> > +
> > +static ssize_t iscsi_tpg_param_store_BlkioCgroup(
> > +	struct se_portal_group *se_tpg,
> > +	const char *page,
> > +	size_t count)
> > +{
> > +	struct iscsi_portal_group *tpg = container_of(se_tpg,
> > +			struct iscsi_portal_group, tpg_se_tpg);
> > +	u32 val;
> > +	int ret;
> > +
> > +	if (iscsit_get_tpg(tpg) < 0)
> > +		return -EINVAL;
> > +
> > +	ret = kstrtou32(page, 0, &val);
> > +	if (ret)
> > +		goto out;
> > +	ret = iscsit_ta_tpg_set_blkcg(tpg, val);
> > +	if (ret < 0)
> > +		goto out;
> > +
> > +	iscsit_put_tpg(tpg);
> > +	return count;
> > +out:
> > +	iscsit_put_tpg(tpg);
> > +	return ret;
> > +}
> > +
> > +TPG_PARAM_ATTR(BlkioCgroup, S_IRUGO | S_IWUSR);
> > +
> >  static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
> >  	&iscsi_tpg_param_AuthMethod.attr,
> >  	&iscsi_tpg_param_HeaderDigest.attr,
> > @@ -1434,6 +1479,7 @@ static struct configfs_attribute *lio_target_tpg_param_attrs[] = {
> >  	&iscsi_tpg_param_OFMarker.attr,
> >  	&iscsi_tpg_param_IFMarkInt.attr,
> >  	&iscsi_tpg_param_OFMarkInt.attr,
> > +	&iscsi_tpg_param_BlkioCgroup.attr,
> >  	NULL,
> >  };
> >  
> > diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
> > index c20b561..4f59416 100644
> > --- a/drivers/target/iscsi/iscsi_target_login.c
> > +++ b/drivers/target/iscsi/iscsi_target_login.c
> > @@ -20,6 +20,7 @@
> >  #include <linux/string.h>
> >  #include <linux/kthread.h>
> >  #include <linux/idr.h>
> > +#include <linux/cgroup.h>
> >  #include <scsi/iscsi_proto.h>
> >  #include <target/target_core_base.h>
> >  #include <target/target_core_fabric.h>
> > @@ -712,8 +713,18 @@ static void iscsi_post_login_start_timers(struct iscsi_conn *conn)
> >  
> >  int iscsit_start_kthreads(struct iscsi_conn *conn)
> >  {
> > +	struct iscsi_portal_group *tpg = conn->tpg;
> > +	struct cgroup_subsys_state *blk_css = NULL;
> >  	int ret = 0;
> >  
> > +	if (iscsit_get_tpg(tpg) < 0)
> > +		return -EINVAL;
> > +	if (tpg->blk_css) {
> > +		blk_css = tpg->blk_css;
> > +		css_get(blk_css);
> > +	}
> > +	iscsit_put_tpg(tpg);
> > +
> >  	spin_lock(&iscsit_global->ts_bitmap_lock);
> >  	conn->bitmap_id = bitmap_find_free_region(iscsit_global->ts_bitmap,
> >  					ISCSIT_BITMAP_BITS, get_order(1));
> > @@ -722,7 +733,8 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
> >  	if (conn->bitmap_id < 0) {
> >  		pr_err("bitmap_find_free_region() failed for"
> >  		       " iscsit_start_kthreads()\n");
> > -		return -ENOMEM;
> > +		ret = -ENOMEM;
> > +		goto put_blk_css;
> >  	}
> >  
> >  	conn->tx_thread = kthread_run(iscsi_target_tx_thread, conn,
> > @@ -732,6 +744,11 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
> >  		ret = PTR_ERR(conn->tx_thread);
> >  		goto out_bitmap;
> >  	}
> > +	if (blk_css) {
> > +		ret = cgroup_kernel_attach(blk_css->cgroup, conn->tx_thread);
> > +		if (ret < 0)
> > +			goto out_tx;
> > +	}
> >  	conn->tx_thread_active = true;
> >  
> >  	conn->rx_thread = kthread_run(iscsi_target_rx_thread, conn,
> > @@ -741,9 +758,21 @@ int iscsit_start_kthreads(struct iscsi_conn *conn)
> >  		ret = PTR_ERR(conn->rx_thread);
> >  		goto out_tx;
> >  	}
> > +	if (blk_css) {
> > +		ret = cgroup_kernel_attach(blk_css->cgroup, conn->rx_thread);
> > +		if (ret < 0)
> > +			goto out_rx;
> > +	}
> >  	conn->rx_thread_active = true;
> >  
> > +	if (blk_css)
> > +		css_put(blk_css);
> > +
> >  	return 0;
> > +out_rx:
> > +	send_sig(SIGINT, conn->rx_thread, 1);
> > +	kthread_stop(conn->rx_thread);
> > +	conn->rx_thread_active = false;
> 
> Seems rx_thread_active = false can be skipped here, 
> it is set to true in success case only,
> after goto to this label.

If in a future we will add one more out label, we will need
rx_thread_active = false, so I added it in this patch to reduce a chance
to forget about it next time.

> 
> >  out_tx:
> >  	send_sig(SIGINT, conn->tx_thread, 1);
> >  	kthread_stop(conn->tx_thread);
> > @@ -753,6 +782,9 @@ out_bitmap:
> >  	bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
> >  			      get_order(1));
> >  	spin_unlock(&iscsit_global->ts_bitmap_lock);
> > +put_blk_css:
> > +	if (blk_css)
> > +		css_put(blk_css);
> >  	return ret;
> >  }
> >  
> > diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c
> > index 2426b0b..792d3ee 100644
> > --- a/drivers/target/iscsi/iscsi_target_tpg.c
> > +++ b/drivers/target/iscsi/iscsi_target_tpg.c
> > @@ -16,6 +16,7 @@
> >   * GNU General Public License for more details.
> >   ******************************************************************************/
> >  
> > +#include <linux/cgroup.h>
> >  #include <target/target_core_base.h>
> >  #include <target/target_core_fabric.h>
> >  #include <target/target_core_configfs.h>
> > @@ -303,6 +304,9 @@ int iscsit_tpg_del_portal_group(
> >  	list_del(&tpg->tpg_list);
> >  	spin_unlock(&tiqn->tiqn_tpg_lock);
> >  
> > +	if (tpg->blk_css)
> > +		css_put(tpg->blk_css);
> > +
> >  	pr_debug("CORE[%s]_TPG[%hu] - Deleted iSCSI Target Portal Group\n",
> >  			tiqn->tiqn, tpg->tpgt);
> >  
> > @@ -896,3 +900,35 @@ int iscsit_ta_tpg_enabled_sendtargets(
> >  
> >  	return 0;
> >  }
> > +
> > +int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag)
> > +{
> > +	struct cgroup_subsys_state *css;
> > +
> > +	if (flag != 1)
> > +		return -EINVAL;
> > +
> > +	css = task_get_css(current, blkio_subsys_id);
> > +	if (tpg->blk_css)
> > +		css_put(tpg->blk_css);
> > +	tpg->blk_css = css;
> > +
> > +	return 0;
> > +}
> > +
> > +int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page)
> > +{
> > +	int rb;
> > +
> > +	if (tpg->blk_css) {
> > +		rb = cgroup_path(tpg->blk_css->cgroup, page, PAGE_SIZE - 1);
> > +		if (rb == 0)
> > +			rb = strlen(page);
> > +		page[rb] = '\n';
> > +		page[rb + 1] = 0;
> > +		rb++;
> > +	} else
> > +		rb = 0;
> > +
> > +	return rb;
> > +}
> > diff --git a/drivers/target/iscsi/iscsi_target_tpg.h b/drivers/target/iscsi/iscsi_target_tpg.h
> > index b2f49c0..416b929 100644
> > --- a/drivers/target/iscsi/iscsi_target_tpg.h
> > +++ b/drivers/target/iscsi/iscsi_target_tpg.h
> > @@ -40,5 +40,7 @@ extern int iscsit_ta_demo_mode_discovery(struct iscsi_portal_group *, u32);
> >  extern int iscsit_ta_default_erl(struct iscsi_portal_group *, u32);
> >  extern int iscsit_ta_t10_pi(struct iscsi_portal_group *, u32);
> >  extern int iscsit_ta_tpg_enabled_sendtargets(struct iscsi_portal_group *, u32);
> > +extern int iscsit_ta_tpg_set_blkcg(struct iscsi_portal_group *tpg, u32 flag);
> > +extern int iscsit_ta_tpg_show_blkcg(struct iscsi_portal_group *tpg, char *page);
> >  
> >  #endif /* ISCSI_TARGET_TPG_H */
> > diff --git a/include/target/iscsi/iscsi_target_core.h b/include/target/iscsi/iscsi_target_core.h
> > index 3e3b2f9..83eb63b 100644
> > --- a/include/target/iscsi/iscsi_target_core.h
> > +++ b/include/target/iscsi/iscsi_target_core.h
> > @@ -838,6 +838,8 @@ struct iscsi_portal_group {
> >  	struct iscsi_tiqn	*tpg_tiqn;
> >  	struct list_head	tpg_gnp_list;
> >  	struct list_head	tpg_list;
> > +
> > +	struct cgroup_subsys_state *blk_css;
> >  } ____cacheline_aligned;
> >  
> >  struct iscsi_wwn_stat_grps {
> > diff --git a/kernel/cgroup.c b/kernel/cgroup.c
> > index 11c547f..0fbb37e 100644
> > --- a/kernel/cgroup.c
> > +++ b/kernel/cgroup.c
> > @@ -5924,6 +5924,7 @@ int cgroup_kernel_attach(struct cgroup *cgrp, struct task_struct *tsk)
> >  	mutex_unlock(&cgroup_mutex);
> >  	return ret;
> >  }
> > +EXPORT_SYMBOL_GPL(cgroup_kernel_attach);
> >  
> >  void cgroup_kernel_close(struct cgroup *cgrp)
> >  {
> >