【OLK-6.6】cgroup blkio需要支持父目录配置对子目录生效

[TOC]

#### 原理

##### cgroup_subsys_on_dfl()

该接口用来判断cgroup子系统是否支持层级关系，返回false代表不支持

```c
#define cgroup_subsys_on_dfl(ss)
	static_branch_likely(&ss ## _on_dfl_key)
```

blk throttle会根据该接口的返回值做不同的处理，例如在blk throttle 初始化时的处理：

```c
static void throtl_pd_init(struct blkg_policy_data *pd)
{
        struct throtl_grp *tg = pd_to_tg(pd);
        struct blkcg_gq *blkg = tg_to_blkg(tg);
        struct throtl_data *td = blkg->q->td;
        struct throtl_service_queue *sq = &tg->service_queue;

/*
        ┊* If on the default hierarchy, we switch to properly hierarchical
        ┊* behavior where limits on a given throtl_grp are applied to the
        ┊* whole subtree rather than just the group itself.  e.g. If 16M
        ┊* read_bps limit is set on the root group, the whole system can't
        ┊* exceed 16M for the device.
        ┊*
        ┊* If not on the default hierarchy, the broken flat hierarchy
        ┊* behavior is retained where all throtl_grps are treated as if
        ┊* they're all separate root groups right below throtl_data.
        ┊* Limits of a group don't interact with limits of other groups
        ┊* regardless of the position of the group in the hierarchy.
        ┊*/
        sq->parent_sq = &td->service_queue;
        if (cgroup_subsys_on_dfl(io_cgrp_subsys) && blkg->parent)
                sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue;
        tg->td = td;
}
```

如果cgroup_subsys_on_dfl()返回false，对throttle的配置都只对当前cgroup生效

##### cgroup_mount v1 和 v2区别

v1的cgroup_root是每次挂载申请出来的

v2使用的是全局唯一的cgrp_dfl_root

cgroup_mount ：

```c
if (fs_type == &cgroup2_fs_type) {
	dentry = cgroup_do_mount(&cgroup2_fs_type, flags, &cgrp_dfl_root, 
							 CGROUP2_SUPER_MAGIC, ns);
} else {
	dentry = cgroup1_mount(&cgroup_fs_type, flags, data, CGROUP_SUPER_MAGIC, ns);
}
```

cgroup1_mount：

```c
root = kzalloc(sizeof(*root), GFP_KERNEL);
init_cgroup_root(root, &opts);
cgroup_setup_root(root, opts.subsys_mask);
dentry = cgroup_do_mount(&cgroup_fs_type, flags, root, CGROUP_SUPER_MAGIC, ns);
```

cgroup_setup_root : 初始化 cgroup_subsys_on_dfl_key

```c
 do_each_subsys_mask(ss, ssid, ss_mask) {
 	if (dst_root == &cgrp_dfl_root) { -> v2 使用的是 cgrp_dfl_root
 		static_branch_enable(cgroup_subsys_on_dfl_key[ssid]); -> v2默认打开
 	} else {
 		dcgrp->subtree_control |= 1 << ssid;
 		static_branch_disable(cgroup_subsys_on_dfl_key[ssid]); -> v1默认关闭了
 	}
```

#### 修改

添加系统启动启动参数控制

#### 测试用例

1. 使能iops/bps限制，测试在某个子cgroup下 读/写 限制效果
2. 使能iops/bps限制，同时在多个子cgroup下下IO，测试 读/写 限制效果

```
test_dev=/dev/sda
dev="8:0"
base=/sys/fs/cgroup/blkio/
tmpdir=/tmp/

# $1 cgroup
# $2 iops
set_iops()
{
	echo "----- set cgroup $1 iops $2 -----"
	echo "$dev $2" > $base/$1/blkio.throttle.read_iops_device || return 1
	echo "$dev $2" > $base/$1/blkio.throttle.write_iops_device || return 1

return 0
}

# $1 cgroup
clear_iops()
{
	echo "----- clear cgroup $1 iops -----"
	echo "$dev 0" > $base/$1/blkio.throttle.read_iops_device || return 1
	echo "$dev 0" > $base/$1/blkio.throttle.write_iops_device || return 1

return 0
}

# $1 cgroup
# $2 bps
set_bps()
{
	echo "----- set cgroup $1 bps $2 -----"
	echo "$dev $2" > $base/$1/blkio.throttle.read_bps_device || return 1
	echo "$dev $2" > $base/$1/blkio.throttle.write_bps_device || return 1

return 0
}

# $1 cgroup
clear_bps()
{
	echo "----- clear cgroup $1 bps -----"
	echo "$dev 0" > $base/$1/blkio.throttle.read_bps_device || return 1
	echo "$dev 0" > $base/$1/blkio.throttle.write_bps_device || return 1

return 0
}

# $1 bs
# $2 rw
# $3 iodepth
# $4 cgroup
# $5 result
__fio()
{
	fio \
	-filename=$test_dev \
	-name=test \
	-ioengine=libaio \
	-direct=1 \
	-numjobs=1 \
	-bs=$1 \
	-rw=$2 \
	-iodepth=$3 \
	-ramp_time=5 &> $5
}

__fail()
{
	echo "$@"
	exit 1
}

# $1 cg1
# $2 cg1 iops
# $3 cg1 bps
# $4 cg2
# $5 cg3
# $6 bs
# $7 rw
# $8 iodepth
test()
{
	if [ $2 -eq 0 ] && [ $3 -eq 0 ]; then
		__fail "iops and bps can't both be 0"
	fi

if [ "$4" == "" ]; then
		__fail "cg2 can't be null"
	fi

echo "##### start test #####"
	echo "##### cg1:      $1"
	echo "##### cg1 iops: $2"
	echo "##### cg1 bps   $3"
	echo "##### cg2:      $4"
	echo "##### cg3:      $5"
	echo "##### bs:       $6"
	echo "##### rw:       $7"
	echo "##### iodepth:  $8"
	echo "#####################"

mkdir -p $base/$1 || __fail "creat dir $1 failed"
	mkdir -p $base/$4 || __fail "creat dir $4 failed"
	mkdir -p $base/$5 || __fail "creat dir $5 failed"
	
	if [ $2 -ne 0 ]; then
		set_iops $1 $2 || __fail "set iops $2 to $1 failed"
	fi

if [ $3 -ne 0 ]; then
		set_bps $1 $3 || __fail "set bps $3 to $1 failed"
	fi

echo "----- start fio in cgroup $4 -----"
	__fio $6 $7 $8 $4 "$tmpdir/res1" &
	echo $! > $base/$4/cgroup.procs

if [ "$5" != "" ]; then
		echo "----- start fio in cgroup $5 -----"
		__fio $6 $7 $8 $5 "$tmpdir/res2" &
		echo $! > $base/$5/cgroup.procs
	fi

sleep 30
	pkill -2 fio
	sleep 2
	echo "----- cgroup $4 result -----"
	cat $tmpdir/res1 | grep IOPS
	if [ "$5" != "" ]; then
		echo "----- cgroup $4 result -----"
		cat $tmpdir/res2 | grep IOPS
	fi

clear_iops $1
	clear_bps $1
}

echo "##### test1 iops in child cgroup read"
test "/" 128 0 "d1" "" 4096 read 1

echo "##### test2 iops in child cgroup write"
test "/" 128 0 "d1" "" 4096 write 1

echo "##### test3 bps in child cgroup read"
test "/" 0 1048576 "d1" "" 4096 read 1

echo "##### test4 bps in child cgroup write"
test "/" 0 1048576 "d1" "" 4096 write 1

echo "##### test5 iops in two child cgroup read"
test "/" 128 0 "d1" "d2" 4096 read 1

echo "##### test6 iops in two child cgroup write"
test "/" 128 0 "d1" "d2" 4096 write 1

echo "##### test7 bps in two child cgroup read"
test "/" 0 1048576 "d1" "d2" 4096 read 1

echo "##### test8 bps in two child cgroup write"
test "/" 0 1048576 "d1" "d2" 4096 write 1

```
#### 测试结果
```
root@fedora ~]# sh test.sh                                                                                                                                                          [94/94]
##### test1 iops in child cgroup read
##### start test #####
##### cg1:      /
##### cg1 iops: 128
##### cg1 bps   0
##### cg2:      d1
##### cg3:
##### bs:       4096
##### rw:       read
##### iodepth:  1
#####################
----- set cgroup / iops 128 -----
----- start fio in cgroup d1 -----
----- cgroup d1 result -----
  read: IOPS=120, BW=480KiB/s (492kB/s)(11.1MiB/23698msec)
----- clear cgroup / iops -----
----- clear cgroup / bps -----
##### test2 iops in child cgroup write
##### start test #####
##### cg1:      /
##### cg1 iops: 128
##### cg1 bps   0
##### cg2:      d1
##### cg3:
##### bs:       4096
##### rw:       write
##### iodepth:  1
#####################
----- set cgroup / iops 128 -----
----- start fio in cgroup d1 -----
----- cgroup d1 result -----
  write: IOPS=119, BW=480KiB/s (491kB/s)(11.6MiB/24705msec); 0 zone resets
----- clear cgroup / iops -----
----- clear cgroup / bps -----
##### test3 bps in child cgroup read
##### start test #####
##### cg1:      /
##### cg1 iops: 0
##### cg1 bps   1048576
##### cg2:      d1
##### cg3:
##### bs:       4096
##### rw:       read
##### iodepth:  1
#####################                                                                                                                                                                [49/94]
----- set cgroup / bps 1048576 -----
----- start fio in cgroup d1 -----
----- cgroup d1 result -----
  read: IOPS=256, BW=1024KiB/s (1049kB/s)(24.7MiB/24703msec)
----- clear cgroup / iops -----
----- clear cgroup / bps -----
##### test4 bps in child cgroup write
##### start test #####
##### cg1:      /
##### cg1 iops: 0
##### cg1 bps   1048576
##### cg2:      d1
##### cg3:
##### bs:       4096
##### rw:       write
##### iodepth:  1
#####################
----- set cgroup / bps 1048576 -----
----- start fio in cgroup d1 -----
----- cgroup d1 result -----
  write: IOPS=256, BW=1024KiB/s (1049kB/s)(24.7MiB/24696msec); 0 zone resets
----- clear cgroup / iops -----
----- clear cgroup / bps -----
##### test5 iops in two child cgroup read
##### start test #####
##### cg1:      /
##### cg1 iops: 128
##### cg1 bps   0
##### cg2:      d1
##### cg3:      d2
##### bs:       4096
##### rw:       read
##### iodepth:  1
#####################
----- set cgroup / iops 128 -----
----- start fio in cgroup d1 -----
----- start fio in cgroup d2 -----
----- cgroup d1 result -----
  read: IOPS=99, BW=400KiB/s (409kB/s)(9832KiB/24602msec)
----- cgroup d1 result -----
  read: IOPS=20, BW=80.2KiB/s (82.1kB/s)(1972KiB/24599msec)
----- clear cgroup / iops -----
----- clear cgroup / bps -----
##### test6 iops in two child cgroup write
##### start test #####
##### cg1:      /
##### cg1 iops: 128
##### cg1 bps   0
##### cg2:      d1
##### cg3:      d2
##### bs:       4096
##### rw:       write
##### iodepth:  1
```

GVP openEuler/kernel
Closed

Content Risk Flag

Comments (2)

GVPopenEuler/kernelClosed

Content Risk Flag