diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index a2ae0fbd7c153468ac8fe5243ae8a86a05c9e0ec..3ed2d89ec34347b3422f6bba8478aabf6da27525 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -97,7 +97,11 @@ CONFIG_PREEMPT_NONE=y # CONFIG_PREEMPT_VOLUNTARY is not set # CONFIG_PREEMPT is not set # CONFIG_PREEMPT_DYNAMIC is not set -CONFIG_XCU_SCHEDULER=n +CONFIG_XCU_SCHEDULER=y +CONFIG_XCU_VSTREAM=y +CONFIG_XSCHED_NR_CUS=128 +CONFIG_XCU_SCHED_RT=y +# CONFIG_XCU_SCHED_CFS is not set # # CPU/Task time and stats accounting diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 11bbad302d268a988b1be82c0611687d0e0e89e7..612ef34a6c7344f07ca5a81ee9f63aa92e2159fb 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -117,7 +117,11 @@ CONFIG_PREEMPT_NONE=y # CONFIG_PREEMPT_VOLUNTARY is not set # CONFIG_PREEMPT is not set # CONFIG_PREEMPT_DYNAMIC is not set -CONFIG_XCU_SCHEDULER=n +CONFIG_XCU_SCHEDULER=y +CONFIG_XCU_VSTREAM=y +CONFIG_XSCHED_NR_CUS=128 +CONFIG_XCU_SCHED_RT=y +# CONFIG_XCU_SCHED_CFS is not set # # CPU/Task time and stats accounting diff --git a/drivers/xcu/xcu_group.c b/drivers/xcu/xcu_group.c index 0cd8f535fb2b4a75be6a5e4fe02f4b625becbe97..1cf159b8f57db0d8964b8a88d6051076a6cc9c28 100644 --- a/drivers/xcu/xcu_group.c +++ b/drivers/xcu/xcu_group.c @@ -327,7 +327,9 @@ int xsched_xcu_register(struct xcu_group *group, uint32_t phys_id) return ret; } +#ifdef CONFIG_CGROUP_XCU xcu_cfs_root_cg_init(xcu); +#endif /* CONFIG_CGROUP_XCU */ return 0; } diff --git a/include/linux/vstream.h b/include/linux/vstream.h index fd393ec97a99ab2cd70d8db51828273da41d8f24..7d99e416624fc74106893c205492282fdfad4c4e 100644 --- a/include/linux/vstream.h +++ b/include/linux/vstream.h @@ -39,8 +39,8 @@ typedef struct vstream_metadata { typedef int vstream_manage_t(struct vstream_args *arg); typedef struct vstream_info { - uint32_t user_stream_id; uint32_t id; + uint32_t sq_id; uint32_t vcq_id; uint32_t logic_vcq_id; uint32_t dev_id; diff --git a/include/linux/xsched.h b/include/linux/xsched.h index d97b3beae8ad2a6bd547a8540f6efd6b87697d00..38db18c0d5701f5001838c5e17db0b2f10f6270c 100644 --- a/include/linux/xsched.h +++ b/include/linux/xsched.h @@ -11,25 +11,22 @@ #define pr_fmt(fmt) fmt #endif -#define XSCHED_INFO_PREFIX "XSched [INFO]: " +#define XSCHED_LOG_PREFIX "XSched" #define XSCHED_INFO(fmt, ...) \ - pr_info(pr_fmt(XSCHED_INFO_PREFIX fmt), ##__VA_ARGS__) + pr_info(pr_fmt(XSCHED_LOG_PREFIX " [INFO]: " fmt), ##__VA_ARGS__) -#define XSCHED_ERR_PREFIX "XSched [ERROR]: " #define XSCHED_ERR(fmt, ...) \ - pr_err(pr_fmt(XSCHED_ERR_PREFIX fmt), ##__VA_ARGS__) + pr_err(pr_fmt(XSCHED_LOG_PREFIX " [ERROR]: " fmt), ##__VA_ARGS__) -#define XSCHED_WARN_PREFIX "XSched [WARNING]: " #define XSCHED_WARN(fmt, ...) \ - pr_warn(pr_fmt(XSCHED_WARN_PREFIX fmt), ##__VA_ARGS__) + pr_warn(pr_fmt(XSCHED_LOG_PREFIX " [WARNING]: " fmt), ##__VA_ARGS__) /* * Debug specific prints for XSched */ -#define XSCHED_DEBUG_PREFIX "XSched [DEBUG]: " #define XSCHED_DEBUG(fmt, ...) \ - pr_debug(pr_fmt(XSCHED_DEBUG_PREFIX fmt), ##__VA_ARGS__) + pr_debug(pr_fmt(XSCHED_LOG_PREFIX " [DEBUG]: " fmt), ##__VA_ARGS__) #define XSCHED_CALL_STUB() \ XSCHED_DEBUG(" -----* %s @ %s called *-----\n", __func__, __FILE__) @@ -195,7 +192,7 @@ struct xsched_entity { pid_t tgid; /* Amount of pending kicks currently sitting on this context. */ - atomic_t kicks_pending_ctx_cnt; + atomic_t kicks_pending_cnt; /* Amount of submitted kicks context, used for resched decision. */ atomic_t submitted_one_kick; @@ -229,10 +226,12 @@ struct xsched_entity { */ struct xsched_cu *xcu; +#ifdef CONFIG_CGROUP_XCU /* Link to list of xsched_group items */ struct list_head group_node; struct xsched_group *parent_grp; bool is_group; +#endif /* CONFIG_CGROUP_XCU */ /* General purpose xse lock. */ spinlock_t xse_lock; @@ -246,6 +245,7 @@ struct xcg_attach_entry { struct list_head node; }; +#ifdef CONFIG_CGROUP_XCU /* xsched_group's xcu related stuff */ struct xsched_group_xcu_priv { /* Owner of this group */ @@ -309,22 +309,18 @@ struct xsched_group { struct cgroup_file xcu_file[NR_XCU_FILE_TYPES]; struct work_struct file_show_work; }; - -#define XSCHED_RQ_OF(xse) \ - (container_of(((xse)->cfs.cfs_rq), struct xsched_rq, cfs)) - -#define XSCHED_RQ_OF_CFS_XSE(cfs_xse) \ - (container_of(((cfs_xse)->cfs_rq), struct xsched_rq, cfs)) +#endif /* CONFIG_CGROUP_XCU */ #define XSCHED_SE_OF(cfs_xse) \ (container_of((cfs_xse), struct xsched_entity, cfs)) +#ifdef CONFIG_CGROUP_XCU #define xcg_parent_grp_xcu(xcg) \ ((xcg)->self->parent->perxcu_priv[(xcg)->xcu_id]) #define xse_parent_grp_xcu(xse_cfs) \ (&((XSCHED_SE_OF(xse_cfs) \ - ->parent_grp->perxcu_priv[(XSCHED_SE_OF(xse_cfs))->xcu->id]))) + ->parent_grp->perxcu_priv[(XSCHED_SE_OF(xse_cfs))->xcu->id]))) static inline struct xsched_group_xcu_priv * xse_this_grp_xcu(struct xsched_entity_cfs *xse_cfs) @@ -340,40 +336,7 @@ xse_this_grp(struct xsched_entity_cfs *xse_cfs) { return xse_cfs ? xse_this_grp_xcu(xse_cfs)->self : NULL; } - -/* Increments pending kicks counter for an XCU that the given - * xsched entity is attached to and for xsched entity's xsched - * class. - */ -static inline int xsched_inc_pending_kicks_xse(struct xsched_entity *xse) -{ - atomic_inc(&xse->xcu->pending_kicks); - /* Icrement pending kicks for current XSE. */ - atomic_inc(&xse->kicks_pending_ctx_cnt); - - return 0; -} - -/* Decrements pending kicks counter for an XCU that the given - * xsched entity is attached to and for XSched entity's sched - * class. - */ -static inline int xsched_dec_pending_kicks_xse(struct xsched_entity *xse) -{ - atomic_dec(&xse->xcu->pending_kicks); - /* Decrementing pending kicks for current XSE. */ - atomic_dec(&xse->kicks_pending_ctx_cnt); - - return 0; -} - -/* Checks if there are pending kicks left on a given XCU for all - * xsched classes. - */ -static inline bool xsched_check_pending_kicks_xcu(struct xsched_cu *xcu) -{ - return atomic_read(&xcu->pending_kicks); -} +#endif /* CONFIG_CGROUP_XCU */ static inline int xse_integrity_check(const struct xsched_entity *xse) { @@ -498,6 +461,7 @@ void enqueue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); void dequeue_ctx(struct xsched_entity *xse, struct xsched_cu *xcu); int delete_ctx(struct xsched_context *ctx); +#ifdef CONFIG_CGROUP_XCU /* Xsched group manage functions */ void xsched_group_inherit(struct task_struct *tsk, struct xsched_entity *xse); void xcu_cg_subsys_init(void); @@ -511,4 +475,16 @@ void xsched_quota_timeout_update(struct xsched_group *xg); void xsched_quota_account(struct xsched_group *xg, s64 exec_time); bool xsched_quota_exceed(struct xsched_group *xg); void xsched_quota_refill(struct work_struct *work); + +#define XCU_PERIOD_MIN_MS 1 +#define XCU_QUOTA_RUNTIME_INF -1 +#define XCU_SHARES_MIN 1 + +#define XCUCG_SET_FILE_RETRY_COUNT 50 +#define XCUCG_SET_FILE_DELAY_MS 10 + +#define SCHED_CLASS_MAX_LENGTH 4 + +#endif + #endif /* !__LINUX_XSCHED_H__ */ diff --git a/kernel/xsched/Kconfig b/kernel/xsched/Kconfig index b7e7b222c9496bc9429df7e97f21d91ea5e57e80..df6576da88fd6aba9bc333687af296e43ef01d53 100644 --- a/kernel/xsched/Kconfig +++ b/kernel/xsched/Kconfig @@ -5,8 +5,6 @@ config XCU_SCHEDULER default n select XCU_VSTREAM select XCU_SCHED_RT - select XCU_SCHED_CFS - select CGROUP_XCU help This option enables the XSched scheduler, a custom scheduling mechanism designed for heterogeneous compute units (e.g., XPUs). It provides: @@ -56,6 +54,7 @@ config XCU_SCHED_CFS bool "XCU CFS scheduling class" default n depends on XCU_SCHEDULER + select CGROUP_XCU help Enable support for the CFS scheduling class in the XCU scheduler. @@ -72,7 +71,7 @@ config XCU_SCHED_CFS config CGROUP_XCU bool "XCU bandwidth control and group scheduling for xsched_cfs" default n - depends on XCU_SCHEDULER + depends on XCU_SCHEDULER && XCU_SCHED_CFS help This option enables the extended Compute Unit (XCU) resource controller for CFS task groups, providing hierarchical scheduling and fine-grained bandwidth @@ -81,5 +80,14 @@ config CGROUP_XCU - Nested group scheduling with latency isolation - Integration with xsched_cfs for fair CPU resource management + KABI impact: + Enabling this option adds a new xsched entry (SUBSYS(xcu)) into + cgroup_subsys.h, which increases CGROUP_SUBSYS_COUNT in enum + cgroup_subsys_id. This change directly affects the layout of core + kernel structures such as struct css_set (subsys, e_cset_node) and + struct cgroup (subsys, e_csets). Since these structures are widely + referenced by multiple interfaces, enabling this option will + introduce extensive KABI changes. + Required for systems requiring fine-grained resource control in cgroups. If unsure, say N. diff --git a/kernel/xsched/cgroup.c b/kernel/xsched/cgroup.c index f7eeedc80fc38be0acdb13d7f850ca375316d9eb..ea495af71a2248a5b7d67b68d3e0414c66ba8775 100644 --- a/kernel/xsched/cgroup.c +++ b/kernel/xsched/cgroup.c @@ -211,34 +211,8 @@ inline struct xsched_group *xcu_cg_from_css(struct cgroup_subsys_state *css) return css ? container_of(css, struct xsched_group, css) : NULL; } -/* - * Determine whether the given css corresponds to root_xsched_group.css. - * - * Parameter only_css_self: - * - true : Only check whether the css pointer itself is NULL - * (i.e., the subsystem root). Do not dereference xg->parent. - * Used in the allocation path (css_alloc). - * - false : Further check whether the associated xsched_group - * has no parent (i.e., a normal root check). - */ -static inline bool xsched_group_css_is_root(struct cgroup_subsys_state *css, bool only_css_self) +static inline bool xsched_group_is_root(struct xsched_group *xg) { - struct xsched_group *xg; - - /* NULL indicates the subsystem root */ - if (!css) - return true; - - /* - * During the allocation phase, - * cannot find its parent xsched_group via xg->parent, - * so can only determine on the css itself. - */ - if (only_css_self) - return false; - - xg = xcu_cg_from_css(css); - return xg && !xg->parent; } @@ -258,7 +232,7 @@ xcu_css_alloc(struct cgroup_subsys_state *parent_css) { struct xsched_group *xg; - if (xsched_group_css_is_root(parent_css, true)) + if (!parent_css) return &root_xsched_group.css; xg = kmem_cache_alloc(xsched_group_cache, GFP_KERNEL | __GFP_ZERO); @@ -279,20 +253,19 @@ static void xcu_css_free(struct cgroup_subsys_state *css) static void delay_xcu_cg_set_file_show_workfn(struct work_struct *work) { struct xsched_group *xg; - int retry = 50; xg = container_of(work, struct xsched_group, file_show_work); - for (int i = 0; i < retry; i++) { + for (int i = 0; i < XCUCG_SET_FILE_RETRY_COUNT; i++) { if (!xcu_cg_set_file_show(xg)) return; - mdelay(10); + mdelay(XCUCG_SET_FILE_DELAY_MS); } XSCHED_ERR("Failed to control the files xcu.{quota, period, shares} visibility after\n" "%d retries, sched_class=%d, css=0x%lx\n", - retry, xg->sched_class, (uintptr_t)&xg->css); + XCUCG_SET_FILE_RETRY_COUNT, xg->sched_class, (uintptr_t)&xg->css); } static int xcu_css_online(struct cgroup_subsys_state *css) @@ -324,7 +297,7 @@ static void xcu_css_offline(struct cgroup_subsys_state *css) struct xsched_group *xcg; xcg = xcu_cg_from_css(css); - if (!xsched_group_css_is_root(css, false)) { + if (!xsched_group_is_root(xcg)) { switch (xcg->sched_class) { case XSCHED_TYPE_CFS: xcu_cfs_cg_deinit(xcg); @@ -424,20 +397,24 @@ void xcu_move_task(struct task_struct *task, struct xsched_group *old_xcg, struct xsched_cu *xcu; spin_lock(&old_xcg->lock); + list_for_each_entry_safe(xse, tmp, &old_xcg->members, group_node) { if (xse->owner_pid != task_pid_nr(task)) continue; - xcu = xse->xcu; - if (old_xcg != xse->parent_grp) { WARN_ON(old_xcg != xse->parent_grp); + spin_unlock(&old_xcg->lock); return; } + xcu = xse->xcu; + /* delete from the old_xcg */ list_del(&xse->group_node); + spin_unlock(&old_xcg->lock); + mutex_lock(&xcu->xcu_lock); /* dequeue from the current runqueue */ dequeue_ctx(xse, xcu); @@ -446,7 +423,10 @@ void xcu_move_task(struct task_struct *task, struct xsched_group *old_xcg, /* enqueue to the runqueue in new_xcg */ enqueue_ctx(xse, xcu); mutex_unlock(&xcu->xcu_lock); + + return; } + spin_unlock(&old_xcg->lock); } @@ -546,8 +526,8 @@ static ssize_t xcu_sched_class_write(struct kernfs_open_file *of, char *buf, { struct cgroup_subsys_state *css = of_css(of); struct xsched_group *xg = xcu_cg_from_css(css); - char type_name[4]; - int type = -1; + char type_name[SCHED_CLASS_MAX_LENGTH]; + int type; ssize_t ret = sscanf(buf, "%3s", type_name); @@ -565,8 +545,8 @@ static ssize_t xcu_sched_class_write(struct kernfs_open_file *of, char *buf, if (!list_empty(&css->children)) return -EBUSY; - /* only root child can switch scheduler type */ - if (!xg->parent || !xsched_group_css_is_root(&xg->parent->css, false)) + /* only the first level of root can switch scheduler type */ + if (!xsched_group_is_root(xg->parent)) return -EINVAL; ret = xcu_cg_set_sched_class(xg, type); @@ -605,13 +585,14 @@ void xcu_grp_shares_update(struct xsched_group *parent) struct xsched_group *children; u64 rem, sh_sum = 0, sh_gcd = 0, w_gcd = 0, sh_prod_red = 1; - spin_lock(&parent->lock); - list_for_each_entry(children, &(parent)->children_groups, group_node) { + lockdep_assert_held(&cgroup_mutex); + + list_for_each_entry(children, &parent->children_groups, group_node) { if (children->sched_class == XSCHED_TYPE_CFS) sh_gcd = gcd(sh_gcd, children->shares_cfg); } - list_for_each_entry(children, &(parent)->children_groups, group_node) { + list_for_each_entry(children, &parent->children_groups, group_node) { if (children->sched_class == XSCHED_TYPE_CFS) { sh_sum += children->shares_cfg; children->shares_cfg_red = div64_u64(children->shares_cfg, sh_gcd); @@ -622,14 +603,15 @@ void xcu_grp_shares_update(struct xsched_group *parent) } parent->children_shares_sum = sh_sum; - list_for_each_entry(children, &(parent)->children_groups, group_node) { + + list_for_each_entry(children, &parent->children_groups, group_node) { if (children->sched_class == XSCHED_TYPE_CFS) { children->weight = div64_u64(sh_prod_red, children->shares_cfg_red); w_gcd = gcd(w_gcd, children->weight); } } - list_for_each_entry(children, &(parent)->children_groups, group_node) { + list_for_each_entry(children, &parent->children_groups, group_node) { if (children->sched_class == XSCHED_TYPE_CFS) { children->weight = div64_u64(children->weight, w_gcd); for_each_active_xcu(xcu, id) { @@ -639,7 +621,6 @@ void xcu_grp_shares_update(struct xsched_group *parent) } } } - spin_unlock(&parent->lock); } static int xcu_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, @@ -651,7 +632,7 @@ static int xcu_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, switch (cft->private) { case XCU_FILE_PERIOD_MS: - if (val < 1 || val > (S64_MAX / NSEC_PER_MSEC)) { + if (val < XCU_PERIOD_MIN_MS || val > (S64_MAX / NSEC_PER_MSEC)) { ret = -EINVAL; break; } @@ -659,7 +640,7 @@ static int xcu_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, xsched_quota_timeout_update(xcucg); break; case XCU_FILE_QUOTA_MS: - if (val < -1 || val > (S64_MAX / NSEC_PER_MSEC)) { + if (val < XCU_QUOTA_RUNTIME_INF || val > (S64_MAX / NSEC_PER_MSEC)) { ret = -EINVAL; break; } @@ -673,12 +654,14 @@ static int xcu_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, xsched_quota_timeout_update(xcucg); break; case XCU_FILE_SHARES: - if (val <= 0 || val > U64_MAX) { + if (val < XCU_SHARES_MIN || val > U64_MAX) { ret = -EINVAL; break; } + cgroup_lock(); xcucg->shares_cfg = val; xcu_grp_shares_update(xcucg->parent); + cgroup_unlock(); break; default: XSCHED_ERR("invalid operation %lu @ %s\n", cft->private, __func__); diff --git a/kernel/xsched/core.c b/kernel/xsched/core.c index b920a79239998bdbcf0be4ae4b46a2ea3ca2d280..69f1a442f985cc36c39444c8c7c52c5049d536fd 100644 --- a/kernel/xsched/core.c +++ b/kernel/xsched/core.c @@ -39,17 +39,13 @@ static void put_prev_ctx(struct xsched_entity *xse) static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) { - int kick_count, scheduled = 0, not_empty; + int scheduled = 0, not_empty; struct vstream_info *vs; struct xcu_op_handler_params params; struct vstream_metadata *vsm; size_t kick_slice = xse->class->kick_slice; - kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); - XSCHED_DEBUG("Before decrement XSE kick_count=%d @ %s\n", - kick_count, __func__); - - if (kick_count == 0) { + if (atomic_read(&xse->kicks_pending_cnt) == 0) { XSCHED_WARN("Try to select xse that has 0 kicks @ %s\n", __func__); return 0; @@ -58,18 +54,23 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) do { not_empty = 0; for_each_vstream_in_ctx(vs, xse->ctx) { - spin_lock(&vs->stream_lock); vsm = xsched_vsm_fetch_first(vs); - spin_unlock(&vs->stream_lock); + if (!vsm) continue; + list_add_tail(&vsm->node, &xcu->vsm_list); scheduled++; - xsched_dec_pending_kicks_xse(xse); not_empty++; } } while ((scheduled < kick_slice) && (not_empty)); + if (scheduled == 0) + return 0; + + atomic_sub(scheduled, &xse->kicks_pending_cnt); + atomic_add(scheduled, &xcu->pending_kicks); + /* * Iterate over all vstreams in context: * Set wr_cqe bit in last computing task in vsm_list @@ -86,10 +87,6 @@ static size_t select_work_def(struct xsched_cu *xcu, struct xsched_entity *xse) } } - kick_count = atomic_read(&xse->kicks_pending_ctx_cnt); - XSCHED_DEBUG("After decrement XSE kick_count=%d @ %s\n", - kick_count, __func__); - xse->total_scheduled += scheduled; return scheduled; } @@ -107,6 +104,11 @@ static struct xsched_entity *__raw_pick_next_ctx(struct xsched_cu *xcu) scheduled = class->select_work ? class->select_work(xcu, next) : select_work_def(xcu, next); + if (scheduled == 0) { + dequeue_ctx(next, xcu); + return NULL; + } + XSCHED_DEBUG("xse %d scheduled=%zu total=%zu @ %s\n", next->tgid, scheduled, next->total_scheduled, __func__); break; @@ -166,12 +168,8 @@ int delete_ctx(struct xsched_context *ctx) } /* Wait till context has been submitted. */ - while (atomic_read(&xse->kicks_pending_ctx_cnt)) { - XSCHED_DEBUG("Deleting ctx %d, xse->kicks_pending_ctx_cnt=%d @ %s\n", - xse->tgid, atomic_read(&xse->kicks_pending_ctx_cnt), - __func__); + while (atomic_read(&xse->kicks_pending_cnt)) usleep_range(100, 200); - } mutex_lock(&xcu->xcu_lock); if (curr_xse == xse) @@ -179,8 +177,6 @@ int delete_ctx(struct xsched_context *ctx) dequeue_ctx(xse, xcu); --xcu->nr_ctx; mutex_unlock(&xcu->xcu_lock); - XSCHED_DEBUG("Deleting ctx %d, pending kicks left=%d @ %s\n", xse->tgid, - atomic_read(&xse->kicks_pending_ctx_cnt), __func__); xse->class->xse_deinit(xse); @@ -195,6 +191,11 @@ int xsched_xse_set_class(struct xsched_entity *xse) { struct xsched_class *sched = xsched_first_class; + if (!sched) { + XSCHED_ERR("No xsched classes registered @ %s\n", __func__); + return -EINVAL; + } + #ifdef CONFIG_CGROUP_XCU xsched_group_inherit(current, xse); for_each_xsched_class(sched) { @@ -214,7 +215,7 @@ static void submit_kick(struct vstream_metadata *vsm) params.group = vs->xcu->group; params.fd = vs->fd; - params.param_1 = &vs->id; + params.param_1 = &vs->sq_id; params.param_2 = &vs->channel_id; params.param_3 = vsm->sqe; params.param_4 = &vsm->sqe_num; @@ -242,7 +243,7 @@ static void submit_wait(struct vstream_metadata *vsm) params.group = vs->xcu->group; params.param_1 = &vs->channel_id; params.param_2 = &vs->logic_vcq_id; - params.param_3 = &vs->user_stream_id; + params.param_3 = &vs->id; params.param_4 = &vsm->sqe; params.param_5 = vsm->cqe; params.param_6 = vs->drv_ctx; @@ -253,9 +254,6 @@ static void submit_wait(struct vstream_metadata *vsm) XSCHED_ERR("Fail to wait Vstream id %u tasks, logic_cq_id %u.\n", vs->id, vs->logic_vcq_id); } - - XSCHED_DEBUG("Vstream id %u wait finish, logic_cq_id %u\n", - vs->id, vs->logic_vcq_id); } static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) @@ -266,8 +264,6 @@ static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) ktime_t t_start = 0; struct xcu_op_handler_params params; - XSCHED_DEBUG("%s called for xse %d on xcu %u\n", - __func__, xse->tgid, xcu->id); list_for_each_entry_safe(vsm, tmp, &xcu->vsm_list, node) { submit_kick(vsm); XSCHED_DEBUG("Xse %d vsm %u sched_delay: %lld ns\n", @@ -288,8 +284,12 @@ static int __xsched_submit(struct xsched_cu *xcu, struct xsched_entity *xse) kfree(vsm); } + if (submitted == 0) + return 0; + xse->last_exec_runtime += submit_exec_time; xse->total_submitted += submitted; + atomic_sub(submitted, &xcu->pending_kicks); atomic_add(submitted, &xse->submitted_one_kick); INIT_LIST_HEAD(&xcu->vsm_list); XSCHED_DEBUG("Xse %d submitted=%d total=%zu, exec_time=%ld @ %s\n", @@ -315,7 +315,10 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) return -ENOMEM; } + spin_lock(&vs->stream_lock); + if (vs->kicks_count > MAX_VSTREAM_SIZE) { + spin_unlock(&vs->stream_lock); kfree(new_vsm); return -EBUSY; } @@ -323,7 +326,14 @@ int xsched_vsm_add_tail(struct vstream_info *vs, vstream_args_t *arg) xsched_init_vsm(new_vsm, vs, arg); list_add_tail(&new_vsm->node, &vs->metadata_list); new_vsm->add_time = ktime_get(); - vs->kicks_count += 1; + vs->kicks_count++; + + spin_unlock(&vs->stream_lock); + + /* Increasing a total amount of kicks on an CU to which this + * context is attached to based on sched_class. + */ + atomic_inc(&vs->ctx->xse.kicks_pending_cnt); return 0; } @@ -336,17 +346,18 @@ struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs) { struct vstream_metadata *vsm; - if (list_empty(&vs->metadata_list)) { + if (!vs || list_empty(&vs->metadata_list)) { XSCHED_DEBUG("No metadata to fetch from vs %u @ %s\n", vs->id, __func__); return NULL; } + spin_lock(&vs->stream_lock); vsm = list_first_entry(&vs->metadata_list, struct vstream_metadata, node); if (!vsm) { XSCHED_ERR("Corrupted metadata list in vs %u @ %s\n", vs->id, __func__); - return NULL; + goto out_unlock; } list_del(&vsm->node); @@ -356,6 +367,9 @@ struct vstream_metadata *xsched_vsm_fetch_first(struct vstream_info *vs) else vs->kicks_count -= 1; +out_unlock: + spin_unlock(&vs->stream_lock); + return vsm; } @@ -376,11 +390,6 @@ int xsched_schedule(void *input_xcu) break; } - if (!xsched_check_pending_kicks_xcu(xcu)) { - XSCHED_WARN("%s: No pending kicks on xcu %u\n", __func__, xcu->id); - continue; - } - next_xse = __raw_pick_next_ctx(xcu); if (!next_xse) { XSCHED_WARN("%s: Couldn't find next xse on xcu %u\n", __func__, xcu->id); @@ -397,7 +406,7 @@ int xsched_schedule(void *input_xcu) /* if not deleted yet */ put_prev_ctx(curr_xse); - if (!atomic_read(&curr_xse->kicks_pending_ctx_cnt)) + if (!atomic_read(&curr_xse->kicks_pending_cnt)) dequeue_ctx(curr_xse, xcu); #ifdef CONFIG_CGROUP_XCU @@ -460,7 +469,7 @@ int xsched_init_entity(struct xsched_context *ctx, struct vstream_info *vs) int err = 0; struct xsched_entity *xse = &ctx->xse; - atomic_set(&xse->kicks_pending_ctx_cnt, 0); + atomic_set(&xse->kicks_pending_cnt, 0); atomic_set(&xse->submitted_one_kick, 0); xse->total_scheduled = 0; diff --git a/kernel/xsched/vstream.c b/kernel/xsched/vstream.c index 5073c9bf6af5e49ac24610a7edc7d438bda36285..d0815e33e0819d3929d0e55536f7350a852ed856 100644 --- a/kernel/xsched/vstream.c +++ b/kernel/xsched/vstream.c @@ -117,9 +117,6 @@ struct xsched_cu *xcu_find(uint32_t type, return NULL; } - XSCHED_DEBUG("XCU found: type=%u, dev_id=%u, chan_id=%u.\n", - type, dev_id, channel_id); - return group->xcu; } @@ -128,7 +125,7 @@ static int vstream_destroy(vstream_info_t *vstream) int err; struct xsched_context *ctx = NULL; - err = vstream_del(vstream, vstream->id); + err = vstream_del(vstream, vstream->sq_id); if (err) return err; @@ -198,7 +195,6 @@ int ctx_bind_to_xcu(vstream_info_t *vstream_info, struct xsched_context *ctx) ctx->xse.xcu = xcu_found; vstream_info->xcu = xcu_found; revmap_data->dev_id = vstream_info->dev_id; - XSCHED_DEBUG("Ctx bind to xcu %u @ %s\n", xcu_found->id, __func__); hash_add(ctx_revmap, &revmap_data->hash_node, (unsigned long)ctx->dev_id); @@ -252,10 +248,9 @@ static int vstream_bind_to_ctx(struct vstream_info *vs) mutex_lock(&xcu->ctx_list_lock); ctx = ctx_find_by_tgid_and_xcu(vs->tgid, xcu); - if (ctx) { - XSCHED_DEBUG("Ctx %d found @ %s\n", vs->tgid, __func__); + if (ctx) kref_get(&ctx->kref); - } else { + else { err = alloc_ctx_from_vstream(vs, &ctx); if (err) goto out_err; @@ -359,7 +354,7 @@ vstream_get_by_user_stream_id(struct xsched_cu *xcu, uint32_t user_stream_id) mutex_lock(&xcu->vs_array_lock); for (id = 0; id < MAX_VSTREAM_NUM; id++) { if (xcu->vs_array[id] != NULL && - xcu->vs_array[id]->user_stream_id == user_stream_id) { + xcu->vs_array[id]->id == user_stream_id) { ret = xcu->vs_array[id]; break; } @@ -380,8 +375,6 @@ static int vstream_bind_to_xcu(vstream_info_t *vstream_info) /* Bind vstream to a xcu. */ vstream_info->xcu = xcu_found; vstream_info->dev_id = xcu_found->id; - XSCHED_DEBUG("XCU bound to a vstream: type=%u, dev_id=%u, chan_id=%u.\n", - type, vstream_info->dev_id, vstream_info->channel_id); return 0; } @@ -427,10 +420,10 @@ static int sqcq_alloc(struct vstream_args *arg) } vstream->drv_ctx = params.param_5; - vstream->id = sq_id; + vstream->sq_id = sq_id; vstream->vcq_id = cq_id; vstream->logic_vcq_id = logic_cq_id; - vstream->user_stream_id = va_args->user_stream_id; + vstream->id = va_args->user_stream_id; vstream->tgid = tgid; vstream->sqcq_type = va_args->type; ret = vstream_bind_to_ctx(vstream); @@ -447,7 +440,7 @@ static int sqcq_alloc(struct vstream_args *arg) vstream->inode_fd = ret; /* Add new vstream to array after allocating inode */ - ret = vstream_add(vstream, vstream->id); + ret = vstream_add(vstream, vstream->sq_id); if (ret) goto out_err_vstream_file_put; @@ -596,12 +589,10 @@ int vstream_kick(struct vstream_args *arg) do { mutex_lock(&xcu->xcu_lock); - spin_lock(&vstream->stream_lock); /* Adding kick metadata. */ err = xsched_vsm_add_tail(vstream, arg); if (err == -EBUSY) { - spin_unlock(&vstream->stream_lock); mutex_unlock(&xcu->xcu_lock); /* Retry after a while */ @@ -617,14 +608,8 @@ int vstream_kick(struct vstream_args *arg) } enqueue_ctx(xse, xcu); - - /* Increasing a total amount of kicks on an CU to which this - * context is attached to based on sched_class. - */ - xsched_inc_pending_kicks_xse(&vstream->ctx->xse); } while (err == -EBUSY); - spin_unlock(&vstream->stream_lock); mutex_unlock(&xcu->xcu_lock); if (!err) wake_up_interruptible(&xcu->wq_xcu_idle);