[2/3,v2] target: allow to set a blkio cgroup for a backstore

Submitted by Andrei Vagin on May 8, 2018, 7:06 a.m.

Details

Message ID 1525763185-24845-1-git-send-email-avagin@openvz.org
State New
Series "drivers/target: move setting of a blkio cgroup to the backstore level"
Headers show

Commit Message

Andrei Vagin May 8, 2018, 7:06 a.m.
The Block I/O (blkio) subsystem controls and monitors access to I/O on
block devices by tasks in cgroups. With the introduced changes, a
backstore will be like a task in a specified group.

One of interesting feature is an ability to set limits on a number of
I/O operations and bytes per seconds.

A new attribute is added for backstores, it is called blkio_cgroup.

If we write 1 to the attribute file, a blkio cgroup from the current
process is attached to the backstore.

If we write 0 to the attribute file, a current group will be detached
from the backstore.

When we know a blkio cgroup the only thing, what we need to do to make it
work, is to set this group for bio-s.

How to use:
 # Create a test backstore
$ targetcli
targetcli shell version 2.1.fb46
Copyright 2011-2013 by Datera, Inc and others.
/backstores/block> create dev=/dev/loop0 loop0
Created block storage object loop0 using /dev/loop0.
/backstores/block> cd /loopback
/loopback> create
Created target naa.50014056fd3f341c.
/loopback> cd naa.50014056fd3f341c/luns
/loopback/naa...fd3f341c/luns> create /backstores/block/loop0
Created LUN 0.
/loopback/naa...fd3f341c/luns> exit

 # Create a test cgroup and set it to a test backstore
$ CG_PATH=/sys/fs/cgroup/blkio/test
$ BS_PATH=/sys/kernel/config/target/core/iblock_0/loop0/attrib/blkio_cgroup
$ mkdir -p $CG_PATH
$ bash -c "echo 0 > $CG_PATH/tasks && echo 1 > $BS_PATH"
$ cat $BS_PATH
/test

 # Set 6 MB/sec for the backstore
$ echo "7:0 6291456" > $CG_PATH/blkio.throttle.read_bps_device

 # Check that everything work as expected
$ dd if=/dev/sda of=/dev/null iflag=direct bs=1M count=100
100+0 records in
100+0 records out
104857600 bytes (105 MB, 100 MiB) copied, 16.6958 s, 6.3 MB/s

v2: handle errors of cgroup_path()

Signed-off-by: Andrei Vagin <avagin@openvz.org>
---
 drivers/target/target_core_device.c  | 61 ++++++++++++++++++++++++++++++++++++
 drivers/target/target_core_iblock.c  | 38 ++++++++++++++++++++++
 include/target/target_core_backend.h |  4 +++
 include/target/target_core_base.h    |  2 ++
 4 files changed, 105 insertions(+)

Patch hide | download patch | download mbox

diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index cb17aeb..8f47549 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -831,6 +831,61 @@  int se_dev_set_emulate_fua_read(struct se_device *dev, int flag)
 }
 EXPORT_SYMBOL(se_dev_set_emulate_fua_read);
 
+ssize_t se_dev_blkio_cgroup_show(struct se_device *dev, char *page)
+{
+	int rb;
+
+	read_lock(&dev->dev_attrib_lock);
+	if (dev->dev_attrib.blk_css) {
+		rb = cgroup_path(dev->dev_attrib.blk_css->cgroup,
+						page, PAGE_SIZE - 1);
+		if (rb < 0)
+			goto out;
+		if (rb == 0)
+			rb = strlen(page);
+		page[rb] = '\n';
+		page[rb + 1] = 0;
+		rb++;
+	} else
+		rb = 0;
+out:
+	read_unlock(&dev->dev_attrib_lock);
+
+	return rb;
+}
+EXPORT_SYMBOL(se_dev_blkio_cgroup_show);
+
+ssize_t se_dev_blkio_cgroup_store(struct se_device *dev,
+		const char *page, size_t count)
+{
+	struct cgroup_subsys_state *css, *pcss;
+	int ret;
+	u32 val;
+
+	ret = kstrtou32(page, 0, &val);
+	if (ret < 0)
+		return ret;
+
+	if (val > 1)
+		return -EINVAL;
+	if (val == 1)
+		css = task_get_css(current, blkio_subsys_id);
+	else
+		css = NULL;
+
+	write_lock(&dev->dev_attrib_lock);
+	pcss = dev->dev_attrib.blk_css;
+	dev->dev_attrib.blk_css = css;
+	write_unlock(&dev->dev_attrib_lock);
+
+	if (pcss)
+		css_put(pcss);
+
+	return count;
+}
+EXPORT_SYMBOL(se_dev_blkio_cgroup_store);
+
+
 int se_dev_set_emulate_write_cache(struct se_device *dev, int flag)
 {
 	if (flag != 0 && flag != 1) {
@@ -1507,6 +1562,7 @@  struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 	INIT_LIST_HEAD(&dev->state_list);
 	INIT_LIST_HEAD(&dev->qf_cmd_list);
 	INIT_LIST_HEAD(&dev->g_dev_node);
+	rwlock_init(&dev->dev_attrib_lock);
 	spin_lock_init(&dev->execute_task_lock);
 	spin_lock_init(&dev->delayed_cmd_lock);
 	spin_lock_init(&dev->dev_reservation_lock);
@@ -1555,6 +1611,7 @@  struct se_device *target_alloc_device(struct se_hba *hba, const char *name)
 	dev->dev_attrib.unmap_zeroes_data =
 				DA_UNMAP_ZEROES_DATA_DEFAULT;
 	dev->dev_attrib.max_write_same_len = DA_MAX_WRITE_SAME_LEN;
+	dev->dev_attrib.blk_css = NULL;
 
 	xcopy_lun = &dev->xcopy_lun;
 	xcopy_lun->lun_se_dev = dev;
@@ -1728,6 +1785,10 @@  void target_free_device(struct se_device *dev)
 	if (dev->transport->free_prot)
 		dev->transport->free_prot(dev);
 
+	if (dev->dev_attrib.blk_css)
+		css_put(dev->dev_attrib.blk_css);
+	dev->dev_attrib.blk_css = NULL;
+
 	dev->transport->free_device(dev);
 }
 
diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c
index 55be98b..cac5c7b 100644
--- a/drivers/target/target_core_iblock.c
+++ b/drivers/target/target_core_iblock.c
@@ -35,6 +35,7 @@ 
 #include <linux/genhd.h>
 #include <linux/file.h>
 #include <linux/module.h>
+#include <linux/cgroup.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
 #include <asm/unaligned.h>
@@ -680,6 +681,7 @@  iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 	unsigned bio_cnt;
 	int rw = 0;
 	int i;
+	struct cgroup_subsys_state *blk_css = NULL;
 
 	if (data_direction == DMA_TO_DEVICE) {
 		struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
@@ -713,9 +715,17 @@  iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 		return 0;
 	}
 
+	read_lock(&dev->dev_attrib_lock);
+	blk_css = dev->dev_attrib.blk_css;
+	if (blk_css)
+		css_get(blk_css);
+	read_unlock(&dev->dev_attrib_lock);
+
 	bio = iblock_get_bio(cmd, block_lba, sgl_nents);
 	if (!bio)
 		goto fail_free_ibr;
+	if (blk_css)
+		bio_associate_blkcg(bio, blk_css);
 
 	bio_start = bio;
 	bio_list_init(&list);
@@ -740,6 +750,8 @@  iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 			bio = iblock_get_bio(cmd, block_lba, sg_num);
 			if (!bio)
 				goto fail_put_bios;
+			if (blk_css)
+				bio_associate_blkcg(bio, blk_css);
 
 			atomic_inc(&ibr->pending);
 			bio_list_add(&list, bio);
@@ -759,12 +771,18 @@  iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
 
 	iblock_submit_bios(&list, rw);
 	iblock_complete_cmd(cmd);
+
+	if (blk_css)
+		css_put(blk_css);
+
 	return 0;
 
 fail_put_bios:
 	while ((bio = bio_list_pop(&list)))
 		bio_put(bio);
 fail_free_ibr:
+	if (blk_css)
+		css_put(blk_css);
 	kfree(ibr);
 fail:
 	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
@@ -842,6 +860,25 @@  static bool iblock_get_write_cache(struct se_device *dev)
 }
 
 DEF_TB_DEFAULT_ATTRIBS(iblock);
+static ssize_t iblock_dev_show_attr_blkio_cgroup(
+       struct se_dev_attrib *da,
+       char *page)
+{
+       return se_dev_blkio_cgroup_show(da->da_dev, page);
+}
+
+static ssize_t iblock_dev_store_attr_blkio_cgroup(
+       struct se_dev_attrib *da,
+       const char *page,
+       size_t count)
+{
+       return se_dev_blkio_cgroup_store(da->da_dev, page, count);
+}
+
+static struct target_backend_dev_attrib_attribute iblock_dev_attrib_blkio_cgroup =
+		__CONFIGFS_EATTR(blkio_cgroup, S_IRUGO | S_IWUSR,
+		iblock_dev_show_attr_blkio_cgroup,
+		iblock_dev_store_attr_blkio_cgroup);
 
 static struct configfs_attribute *iblock_backend_dev_attrs[] = {
 	&iblock_dev_attrib_emulate_model_alias.attr,
@@ -874,6 +911,7 @@  static struct configfs_attribute *iblock_backend_dev_attrs[] = {
 	&iblock_dev_attrib_unmap_granularity_alignment.attr,
 	&iblock_dev_attrib_unmap_zeroes_data.attr,
 	&iblock_dev_attrib_max_write_same_len.attr,
+	&iblock_dev_attrib_blkio_cgroup.attr,
 	NULL,
 };
 
diff --git a/include/target/target_core_backend.h b/include/target/target_core_backend.h
index 5e1124d..bd2f3fe 100644
--- a/include/target/target_core_backend.h
+++ b/include/target/target_core_backend.h
@@ -143,6 +143,10 @@  int	se_dev_set_max_sectors(struct se_device *, u32);
 int	se_dev_set_optimal_sectors(struct se_device *, u32);
 int	se_dev_set_block_size(struct se_device *, u32);
 
+ssize_t se_dev_blkio_cgroup_show(struct se_device *dev, char *page);
+ssize_t se_dev_blkio_cgroup_store(struct se_device *dev,
+		const char *page, size_t count);
+
 sector_t target_to_linux_sector(struct se_device *dev, sector_t lb);
 bool target_configure_unmap_from_queue(struct se_dev_attrib *attrib,
 				       struct request_queue *q);
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index a72b5a5..8b85104 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -707,6 +707,7 @@  struct se_dev_attrib {
 	u32		max_bytes_per_io;
 	struct se_device *da_dev;
 	struct config_group da_group;
+	struct cgroup_subsys_state *blk_css;
 };
 
 struct se_port_stat_grps {
@@ -806,6 +807,7 @@  struct se_device {
 	atomic_t		dev_ordered_sync;
 	atomic_t		dev_qf_count;
 	int			export_count;
+	rwlock_t		dev_attrib_lock;
 	spinlock_t		delayed_cmd_lock;
 	spinlock_t		execute_task_lock;
 	spinlock_t		dev_reservation_lock;

Comments

Cyrill Gorcunov May 9, 2018, 2:27 p.m.
On Tue, May 08, 2018 at 10:06:25AM +0300, Andrei Vagin wrote:
> The Block I/O (blkio) subsystem controls and monitors access to I/O on
> block devices by tasks in cgroups. With the introduced changes, a
> backstore will be like a task in a specified group.
> 
> One of interesting feature is an ability to set limits on a number of
> I/O operations and bytes per seconds.
> 
> A new attribute is added for backstores, it is called blkio_cgroup.
> 
> If we write 1 to the attribute file, a blkio cgroup from the current
> process is attached to the backstore.
> 
> If we write 0 to the attribute file, a current group will be detached
> from the backstore.
> 
> When we know a blkio cgroup the only thing, what we need to do to make it
> work, is to set this group for bio-s.
> 
> How to use:
>  # Create a test backstore
> $ targetcli
> targetcli shell version 2.1.fb46
> Copyright 2011-2013 by Datera, Inc and others.
> /backstores/block> create dev=/dev/loop0 loop0
> Created block storage object loop0 using /dev/loop0.
> /backstores/block> cd /loopback
> /loopback> create
> Created target naa.50014056fd3f341c.
> /loopback> cd naa.50014056fd3f341c/luns
> /loopback/naa...fd3f341c/luns> create /backstores/block/loop0
> Created LUN 0.
> /loopback/naa...fd3f341c/luns> exit
> 
>  # Create a test cgroup and set it to a test backstore
> $ CG_PATH=/sys/fs/cgroup/blkio/test
> $ BS_PATH=/sys/kernel/config/target/core/iblock_0/loop0/attrib/blkio_cgroup
> $ mkdir -p $CG_PATH
> $ bash -c "echo 0 > $CG_PATH/tasks && echo 1 > $BS_PATH"
> $ cat $BS_PATH
> /test
> 
>  # Set 6 MB/sec for the backstore
> $ echo "7:0 6291456" > $CG_PATH/blkio.throttle.read_bps_device
> 
>  # Check that everything work as expected
> $ dd if=/dev/sda of=/dev/null iflag=direct bs=1M count=100
> 100+0 records in
> 100+0 records out
> 104857600 bytes (105 MB, 100 MiB) copied, 16.6958 s, 6.3 MB/s
> 
> v2: handle errors of cgroup_path()
> 
> Signed-off-by: Andrei Vagin <avagin@openvz.org>
Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>

To cgroup part since I'm not familiar with the rest of the code